Skip to content

Commit 95be852

Browse files
adwk67claude
andcommitted
feat: add merged_overrides() to AirflowCluster
Computes the merged env and webserver_config.py overrides directly from the CRD (role <- role-group), replacing the product-config override path. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent aa855ab commit 95be852

1 file changed

Lines changed: 134 additions & 4 deletions

File tree

  • rust/operator-binary/src/crd

rust/operator-binary/src/crd/mod.rs

Lines changed: 134 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use std::collections::{BTreeMap, BTreeSet};
1+
use std::collections::{BTreeMap, BTreeSet, HashMap};
22

33
use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType};
44
use serde::{Deserialize, Serialize};
@@ -19,7 +19,7 @@ use stackable_operator::{
1919
fragment::{self, Fragment, ValidationError},
2020
merge::Merge,
2121
},
22-
config_overrides::KeyValueConfigOverrides,
22+
config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider},
2323
crd::git_sync,
2424
deep_merger::ObjectOverrides,
2525
k8s_openapi::{
@@ -114,7 +114,7 @@ pub struct AirflowConfigOverrides {
114114
pub webserver_config_py: Option<KeyValueConfigOverrides>,
115115
}
116116

117-
impl stackable_operator::config_overrides::KeyValueOverridesProvider for AirflowConfigOverrides {
117+
impl KeyValueOverridesProvider for AirflowConfigOverrides {
118118
fn get_key_value_overrides(&self, file: &str) -> BTreeMap<String, Option<String>> {
119119
match file {
120120
AIRFLOW_CONFIG_FILENAME => self
@@ -127,6 +127,12 @@ impl stackable_operator::config_overrides::KeyValueOverridesProvider for Airflow
127127
}
128128
}
129129

130+
#[derive(Clone, Debug)]
131+
pub struct MergedOverrides {
132+
pub env_overrides: HashMap<String, String>,
133+
pub config_file_overrides: BTreeMap<String, String>,
134+
}
135+
130136
#[derive(Snafu, Debug)]
131137
pub enum Error {
132138
#[snafu(display("Unknown Airflow role found {role}. Should be one of {roles:?}"))]
@@ -450,6 +456,39 @@ impl v1alpha2::AirflowCluster {
450456
format!("{}-executor-pod-template", self.name_any())
451457
}
452458

459+
pub fn merged_overrides(
460+
&self,
461+
role: &AirflowRole,
462+
rolegroup_name: &str,
463+
) -> Result<MergedOverrides, Error> {
464+
let role_config = role.role_config(self)?;
465+
466+
let mut env_overrides = role_config.config.env_overrides.clone();
467+
let mut file_overrides = role_config
468+
.config
469+
.config_overrides
470+
.get_key_value_overrides(AIRFLOW_CONFIG_FILENAME);
471+
472+
if let Some(rg) = role_config.role_groups.get(rolegroup_name) {
473+
env_overrides.extend(rg.config.env_overrides.clone());
474+
let rg_file = rg
475+
.config
476+
.config_overrides
477+
.get_key_value_overrides(AIRFLOW_CONFIG_FILENAME);
478+
file_overrides.extend(rg_file);
479+
}
480+
481+
let config_file_overrides = file_overrides
482+
.into_iter()
483+
.filter_map(|(k, v)| v.map(|v| (k, v)))
484+
.collect();
485+
486+
Ok(MergedOverrides {
487+
env_overrides,
488+
config_file_overrides,
489+
})
490+
}
491+
453492
/// Retrieve and merge resource configs for role and role groups
454493
pub fn merged_config(
455494
&self,
@@ -1114,13 +1153,15 @@ pub fn build_recommended_labels<'a, T>(
11141153

11151154
#[cfg(test)]
11161155
mod tests {
1156+
use std::collections::BTreeMap;
1157+
11171158
use indoc::formatdoc;
11181159
use stackable_operator::{
11191160
commons::product_image_selection::ResolvedProductImage,
11201161
versioned::test_utils::RoundtripTestData,
11211162
};
11221163

1123-
use crate::{v1alpha1, v1alpha2};
1164+
use crate::{crd::AirflowRole, v1alpha1, v1alpha2};
11241165

11251166
#[test]
11261167
fn test_cluster_config() {
@@ -1345,4 +1386,93 @@ mod tests {
13451386
"#
13461387
}
13471388
}
1389+
1390+
#[test]
1391+
fn merged_overrides_match_config_overrides_from_crd() {
1392+
let cluster_yaml = r#"
1393+
apiVersion: airflow.stackable.tech/v1alpha2
1394+
kind: AirflowCluster
1395+
metadata:
1396+
name: airflow
1397+
spec:
1398+
image:
1399+
productVersion: 3.1.6
1400+
clusterConfig:
1401+
loadExamples: false
1402+
exposeConfig: false
1403+
credentialsSecretName: airflow-admin-credentials
1404+
metadataDatabase:
1405+
postgresql:
1406+
host: airflow-postgresql
1407+
database: airflow
1408+
credentialsSecretName: airflow-postgresql-credentials
1409+
webservers:
1410+
config: {}
1411+
configOverrides:
1412+
webserver_config.py:
1413+
AUTH_TYPE: "AUTH_OID"
1414+
ROLE_ONLY_KEY: "role-value"
1415+
envOverrides:
1416+
ROLE_ENV_VAR: "role-env-value"
1417+
roleGroups:
1418+
default:
1419+
config: {}
1420+
configOverrides:
1421+
webserver_config.py:
1422+
AUTH_TYPE: "AUTH_DB"
1423+
GROUP_ONLY_KEY: "group-value"
1424+
envOverrides:
1425+
GROUP_ENV_VAR: "group-env-value"
1426+
schedulers:
1427+
config: {}
1428+
roleGroups:
1429+
default:
1430+
config: {}
1431+
kubernetesExecutors:
1432+
config: {}
1433+
"#;
1434+
1435+
let deserializer = serde_yaml::Deserializer::from_str(cluster_yaml);
1436+
let cluster: v1alpha2::AirflowCluster =
1437+
serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap();
1438+
1439+
// Webservers/default: role-group overrides merge with (and override) role-level ones
1440+
let overrides = cluster
1441+
.merged_overrides(&AirflowRole::Webserver, "default")
1442+
.expect("merged_overrides should succeed");
1443+
1444+
// configOverrides: group AUTH_TYPE overrides role AUTH_TYPE, both unique keys kept
1445+
assert_eq!(
1446+
overrides.config_file_overrides,
1447+
BTreeMap::from([
1448+
("AUTH_TYPE".into(), "AUTH_DB".into()),
1449+
("ROLE_ONLY_KEY".into(), "role-value".into()),
1450+
("GROUP_ONLY_KEY".into(), "group-value".into()),
1451+
])
1452+
);
1453+
1454+
// envOverrides: both role and group env vars present
1455+
assert_eq!(overrides.env_overrides.len(), 2);
1456+
assert_eq!(
1457+
overrides.env_overrides.get("ROLE_ENV_VAR").unwrap(),
1458+
"role-env-value"
1459+
);
1460+
assert_eq!(
1461+
overrides.env_overrides.get("GROUP_ENV_VAR").unwrap(),
1462+
"group-env-value"
1463+
);
1464+
1465+
// Schedulers/default: no overrides configured → both maps empty
1466+
let overrides = cluster
1467+
.merged_overrides(&AirflowRole::Scheduler, "default")
1468+
.expect("merged_overrides should succeed");
1469+
assert!(
1470+
overrides.config_file_overrides.is_empty(),
1471+
"scheduler should have no config file overrides"
1472+
);
1473+
assert!(
1474+
overrides.env_overrides.is_empty(),
1475+
"scheduler should have no env overrides"
1476+
);
1477+
}
13481478
}

0 commit comments

Comments
 (0)