diff --git a/Cargo.lock b/Cargo.lock index 9263cd2c55..9238877ab0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2898,6 +2898,7 @@ dependencies = [ "byteorder", "bytes", "cc", + "cfg_aliases", "chrono", "clap", "codspeed-divan-compat", diff --git a/Cargo.toml b/Cargo.toml index e35ea56ae3..35d446f385 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,6 +136,9 @@ rouille = { version = "3.6", optional = true, default-features = false, features syslog = { version = "7", optional = true } version-compare = { version = "0.1.1", optional = true } +[build-dependencies] +cfg_aliases = "0.2.1" + [dev-dependencies] assert_cmd = "2.0.13" cc = "1.0" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000000..dd71becafc --- /dev/null +++ b/build.rs @@ -0,0 +1,33 @@ +use cfg_aliases::cfg_aliases; + +fn main() { + cfg_aliases! { + // HTTP-based remote cache backends (excludes memcached and redis) + any_http_remote: { + any( + feature = "azure", + feature = "gcs", + feature = "gha", + feature = "s3", + feature = "webdav", + feature = "oss", + feature = "cos" + ) + }, + // All remote cache backends + any_cache_remote: { + any( + any_http_remote, + feature = "memcached", + feature = "redis" + ) + }, + // Distributed compilation features + any_dist: { + any( + feature = "dist-client", + feature = "dist-server" + ) + }, + } +} diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 24bc4379fe..a16e9a0665 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -31,17 +31,7 @@ use crate::cache::oss::OSSCache; use crate::cache::redis::RedisCache; #[cfg(feature = "s3")] use crate::cache::s3::S3Cache; -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] +#[cfg(any_cache_remote)] use crate::cache::utils::normalize_key; #[cfg(feature = "webdav")] use crate::cache::webdav::WebdavCache; @@ -175,34 +165,14 @@ pub trait Storage: Send + Sync { } /// Wrapper for opendal::Operator that adds basedirs support -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] +#[cfg(any_cache_remote)] pub struct RemoteStorage { operator: opendal::Operator, basedirs: Vec>, rw_mode: CacheMode, } -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] +#[cfg(any_cache_remote)] impl RemoteStorage { pub fn new(operator: opendal::Operator, basedirs: Vec>, rw_mode: CacheMode) -> Self { Self { @@ -214,17 +184,7 @@ impl RemoteStorage { } /// Implement storage for operator. -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] +#[cfg(any_cache_remote)] #[async_trait] impl Storage for RemoteStorage { async fn get(&self, key: &str) -> Result { @@ -380,17 +340,7 @@ impl Storage for RemoteStorage { /// Build a single cache storage from CacheType /// Helper function used by storage_from_config for both single and multi-level caches -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] +#[cfg(any_cache_remote)] pub fn build_single_cache( cache_type: &CacheType, basedirs: &[Vec], @@ -601,17 +551,7 @@ pub fn storage_from_config( } // Single cache or fallback to disk (backward compatible path) - #[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" - ))] + #[cfg(any_cache_remote)] if let Some(cache_type) = &config.cache { debug!("Configuring single cache from CacheType"); return build_single_cache(cache_type, &config.basedirs, pool); diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 9959d21732..b59e73bfb3 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -41,15 +41,7 @@ pub(crate) mod utils; #[cfg(feature = "webdav")] pub mod webdav; -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] +#[cfg(any_http_remote)] pub(crate) mod http_client; pub use crate::cache::cache::*; diff --git a/src/cache/multilevel.rs b/src/cache/multilevel.rs index 7c75bda3f9..c8f5df9b24 100644 --- a/src/cache/multilevel.rs +++ b/src/cache/multilevel.rs @@ -20,33 +20,11 @@ use std::time::{Duration, Instant}; use async_trait::async_trait; use serde::{Deserialize, Serialize}; -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] +#[cfg(any_cache_remote)] use crate::cache::build_single_cache; use crate::cache::disk::DiskCache; use crate::cache::{Cache, CacheMode, CacheWrite, Storage}; use crate::compiler::PreprocessorCacheEntry; -#[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" -))] -use crate::config::CacheType; use crate::config::{Config, PreprocessorCacheModeConfig, WriteErrorPolicy}; use crate::errors::*; @@ -445,113 +423,25 @@ impl MultiLevelStorage { trace!("Added disk cache level"); } else { // Build remote cache - get the appropriate CacheType - #[cfg(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" - ))] + #[cfg(any_cache_remote)] { - let cache_type = match level_name.to_lowercase().as_str() { - #[cfg(feature = "s3")] - "s3" => config.cache_configs.s3.clone().map(CacheType::S3), - #[cfg(not(feature = "s3"))] - "s3" => return Err(anyhow!("Cache level 's3' requires the 's3' feature")), - #[cfg(feature = "redis")] - "redis" => config.cache_configs.redis.clone().map(CacheType::Redis), - #[cfg(not(feature = "redis"))] - "redis" => { - return Err(anyhow!( - "Cache level 'redis' requires the 'redis' feature" - )); - } - #[cfg(feature = "memcached")] - "memcached" => config - .cache_configs - .memcached - .clone() - .map(CacheType::Memcached), - #[cfg(not(feature = "memcached"))] - "memcached" => { - return Err(anyhow!( - "Cache level 'memcached' requires the 'memcached' feature" - )); - } - #[cfg(feature = "gcs")] - "gcs" => config.cache_configs.gcs.clone().map(CacheType::GCS), - #[cfg(not(feature = "gcs"))] - "gcs" => { - return Err(anyhow!("Cache level 'gcs' requires the 'gcs' feature")); - } - #[cfg(feature = "gha")] - "gha" => config.cache_configs.gha.clone().map(CacheType::GHA), - #[cfg(not(feature = "gha"))] - "gha" => { - return Err(anyhow!("Cache level 'gha' requires the 'gha' feature")); - } - #[cfg(feature = "azure")] - "azure" => config.cache_configs.azure.clone().map(CacheType::Azure), - #[cfg(not(feature = "azure"))] - "azure" => { - return Err(anyhow!( - "Cache level 'azure' requires the 'azure' feature" - )); - } - #[cfg(feature = "webdav")] - "webdav" => config.cache_configs.webdav.clone().map(CacheType::Webdav), - #[cfg(not(feature = "webdav"))] - "webdav" => { - return Err(anyhow!( - "Cache level 'webdav' requires the 'webdav' feature" - )); - } - #[cfg(feature = "oss")] - "oss" => config.cache_configs.oss.clone().map(CacheType::OSS), - #[cfg(not(feature = "oss"))] - "oss" => { - return Err(anyhow!("Cache level 'oss' requires the 'oss' feature")); - } - #[cfg(feature = "cos")] - "cos" => config.cache_configs.cos.clone().map(CacheType::COS), - #[cfg(not(feature = "cos"))] - "cos" => { - return Err(anyhow!("Cache level 'cos' requires the 'cos' feature")); - } - _ => { - return Err(anyhow!("Unknown cache level: '{}'", level_name)); - } - }; - - if let Some(cache_type) = cache_type { - let storage = build_single_cache(&cache_type, &config.basedirs, pool) - .with_context(|| { - format!("Failed to build cache for level '{}'", level_name) - })?; - storages.push(storage); - trace!("Added cache level: {}", level_name); - } else { - return Err(anyhow!( + let level_lower = level_name.to_lowercase(); + let (_display_name, cache_type) = + config.cache_configs.cache_type_by_name(&level_lower)?; + let cache_type = cache_type.ok_or_else(|| { + anyhow!( "Cache level '{}' specified in SCCACHE_MULTILEVEL_CHAIN but not configured (missing environment variables)", level_name - )); - } + ) + })?; + let storage = build_single_cache(&cache_type, &config.basedirs, pool) + .with_context(|| { + format!("Failed to build cache for level '{}'", level_name) + })?; + storages.push(storage); + trace!("Added cache level: {}", level_name); } - #[cfg(not(any( - feature = "azure", - feature = "gcs", - feature = "gha", - feature = "memcached", - feature = "redis", - feature = "s3", - feature = "webdav", - feature = "oss", - feature = "cos" - )))] + #[cfg(not(any_cache_remote))] { return Err(anyhow!( "Cache level '{}' requires a backend feature to be enabled (e.g., --features redis,s3)", diff --git a/src/config.rs b/src/config.rs index f6c3660efc..b9a93569a8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -18,7 +18,7 @@ use crate::util::normalize_win_path; use directories::ProjectDirs; use fs::File; use fs_err as fs; -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] use serde::ser::Serializer; use serde::{ Deserialize, Serialize, @@ -178,10 +178,10 @@ pub fn parse_size(val: &str) -> Option { u64::from_str(val).ok().map(|size| size * multiplier) } -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] #[derive(Clone, Debug, PartialEq, Eq)] pub struct HTTPUrl(reqwest::Url); -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] impl Serialize for HTTPUrl { fn serialize(&self, serializer: S) -> StdResult where @@ -190,7 +190,7 @@ impl Serialize for HTTPUrl { serializer.serialize_str(self.0.as_str()) } } -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] impl<'a> Deserialize<'a> for HTTPUrl { fn deserialize(deserializer: D) -> StdResult where @@ -202,7 +202,7 @@ impl<'a> Deserialize<'a> for HTTPUrl { Ok(HTTPUrl(url)) } } -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] fn parse_http_url(url: &str) -> Result { use std::net::SocketAddr; let url = if let Ok(sa) = url.parse::() { @@ -220,7 +220,7 @@ fn parse_http_url(url: &str) -> Result { } Ok(url) } -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] impl HTTPUrl { pub fn from_url(u: reqwest::Url) -> Self { HTTPUrl(u) @@ -556,6 +556,40 @@ impl CacheConfigs { (cache_type, fallback) } + /// Look up a remote cache type by name (e.g. "s3", "gha"). + /// Returns `Ok((display_name, Some(..)))` if the name is known and configured, + /// `Ok((display_name, None))` if known but not configured, `Err` if unknown. + pub fn cache_type_by_name(&self, name: &str) -> Result<(&'static str, Option)> { + match name { + #[cfg(feature = "s3")] + "s3" => Ok(("S3", self.s3.clone().map(CacheType::S3))), + #[cfg(feature = "redis")] + "redis" => Ok(("Redis", self.redis.clone().map(CacheType::Redis))), + #[cfg(feature = "memcached")] + "memcached" => Ok(( + "Memcached", + self.memcached.clone().map(CacheType::Memcached), + )), + #[cfg(feature = "gcs")] + "gcs" => Ok(("GCS", self.gcs.clone().map(CacheType::GCS))), + #[cfg(feature = "gha")] + "gha" => Ok(("GHA", self.gha.clone().map(CacheType::GHA))), + #[cfg(feature = "azure")] + "azure" => Ok(("Azure", self.azure.clone().map(CacheType::Azure))), + #[cfg(feature = "webdav")] + "webdav" => Ok(("WebDAV", self.webdav.clone().map(CacheType::Webdav))), + #[cfg(feature = "oss")] + "oss" => Ok(("OSS", self.oss.clone().map(CacheType::OSS))), + #[cfg(feature = "cos")] + "cos" => Ok(("COS", self.cos.clone().map(CacheType::COS))), + _ => bail!( + "Unknown cache level: '{}' (may require a feature flag, e.g. --features {})", + name, + name + ), + } + } + /// Get ordered list of cache types based on configured levels. /// If levels are specified, returns them in order with validation. /// If no levels specified and single remote cache, returns that single cache. @@ -566,42 +600,17 @@ impl CacheConfigs { let mut caches = Vec::new(); for level_name in &ml_config.chain { let level_name = level_name.trim(); - let cache_type = match level_name { - "s3" => self.s3.clone().map(CacheType::S3).ok_or_else(|| { - anyhow!("S3 cache not configured but specified in levels") - })?, - "redis" => self.redis.clone().map(CacheType::Redis).ok_or_else(|| { - anyhow!("Redis cache not configured but specified in levels") - })?, - "memcached" => self - .memcached - .clone() - .map(CacheType::Memcached) - .ok_or_else(|| { - anyhow!("Memcached cache not configured but specified in levels") - })?, - "gcs" => self.gcs.clone().map(CacheType::GCS).ok_or_else(|| { - anyhow!("GCS cache not configured but specified in levels") - })?, - "gha" => self.gha.clone().map(CacheType::GHA).ok_or_else(|| { - anyhow!("GHA cache not configured but specified in levels") - })?, - "azure" => self.azure.clone().map(CacheType::Azure).ok_or_else(|| { - anyhow!("Azure cache not configured but specified in levels") - })?, - "webdav" => self.webdav.clone().map(CacheType::Webdav).ok_or_else(|| { - anyhow!("Webdav cache not configured but specified in levels") - })?, - "oss" => self.oss.clone().map(CacheType::OSS).ok_or_else(|| { - anyhow!("OSS cache not configured but specified in levels") - })?, - "disk" => { - // Disk cache is handled separately in MultiLevelStorage::from_config - // Mark it by continuing - it will be added to the storage list there - continue; - } - _ => bail!("Unknown cache level: {}", level_name), - }; + if level_name == "disk" { + // Disk cache is handled separately in MultiLevelStorage::from_config + continue; + } + let (display_name, cache_type) = self.cache_type_by_name(level_name)?; + let cache_type = cache_type.ok_or_else(|| { + anyhow!( + "{} cache not configured but specified in levels", + display_name + ) + })?; caches.push(cache_type); } Ok(caches) @@ -754,9 +763,9 @@ impl Default for DistAuth { #[serde(deny_unknown_fields)] pub struct DistConfig { pub auth: DistAuth, - #[cfg(any(feature = "dist-client", feature = "dist-server"))] + #[cfg(any_dist)] pub scheduler_url: Option, - #[cfg(not(any(feature = "dist-client", feature = "dist-server")))] + #[cfg(not(any_dist))] pub scheduler_url: Option, pub cache_dir: PathBuf, pub toolchains: Vec, @@ -2558,13 +2567,13 @@ key_prefix = "cosprefix" auth: DistAuth::Token { token: "secrettoken".to_owned() }, - #[cfg(any(feature = "dist-client", feature = "dist-server"))] + #[cfg(any_dist)] scheduler_url: Some( parse_http_url("http://1.2.3.4:10600") .map(|url| { HTTPUrl::from_url(url) }) .expect("Scheduler url must be valid url str") ), - #[cfg(not(any(feature = "dist-client", feature = "dist-server")))] + #[cfg(not(any_dist))] scheduler_url: Some("http://1.2.3.4:10600".to_owned()), cache_dir: PathBuf::from("/home/user/.cache/sccache-dist-client"), toolchains: vec![], @@ -3067,10 +3076,33 @@ fn test_get_cache_levels_missing_config() { let result = configs.get_cache_levels(); assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + // With the s3 feature, we get "not configured"; without it, "unknown cache level" assert!( - result - .unwrap_err() - .to_string() - .contains("S3 cache not configured") + err.contains("S3 cache not configured") || err.contains("Unknown cache level"), + "unexpected error: {err}" ); } + +#[test] +#[cfg(not(any_cache_remote))] +fn test_cache_type_by_name_all_unknown_without_features() { + let configs = CacheConfigs::default(); + for name in [ + "s3", + "redis", + "memcached", + "gcs", + "gha", + "azure", + "webdav", + "oss", + "cos", + ] { + let err = configs.cache_type_by_name(name).unwrap_err().to_string(); + assert!( + err.contains("Unknown cache level") && err.contains("--features"), + "expected feature hint for '{name}': {err}" + ); + } +} diff --git a/src/dist/mod.rs b/src/dist/mod.rs index 6bc1024aa8..45643b815d 100644 --- a/src/dist/mod.rs +++ b/src/dist/mod.rs @@ -28,16 +28,16 @@ use std::sync::Mutex; use crate::errors::*; -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] mod cache; #[cfg(feature = "dist-client")] pub mod client_auth; -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] pub mod http; #[cfg(test)] mod test; -#[cfg(any(feature = "dist-client", feature = "dist-server"))] +#[cfg(any_dist)] pub use crate::dist::cache::TcCache; // TODO: paths (particularly outputs, which are accessed by an unsandboxed program) diff --git a/src/util.rs b/src/util.rs index bc64c3d1b7..402290233a 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1000,7 +1000,7 @@ pub fn daemonize() -> Result<()> { /// --- /// /// More details could be found at https://github.com/mozilla/sccache/pull/1563 -#[cfg(any(feature = "dist-server", feature = "dist-client"))] +#[cfg(any_dist)] pub fn new_reqwest_blocking_client() -> reqwest::blocking::Client { reqwest::blocking::Client::builder() .pool_max_idle_per_host(0)