refactor: resolve namenode/journalnode pod refs during validation

Move the pod_refs lookups for namenodes and journalnodes out of
reconcile_hdfs and into validate_cluster, storing the results on
ValidatedCluster. Build steps that already receive the validated
cluster (config_map and hdfs_site) now read the refs from it instead
of taking them as separate parameters.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: maltesander

rust/operator-binary/src/controller/build/config_map.rs

@@ -16,7 +16,7 @@ use crate::{
         ConfigFileName, core_site, hadoop_policy, hdfs_site, security_properties, ssl_client,
         ssl_server,
     },
-    crd::{HdfsNodeRole, HdfsPodRef, v1alpha1},
+    crd::{HdfsNodeRole, v1alpha1},
     hdfs_controller::ValidatedCluster,
     product_logging::extend_role_group_config_map,
 };
@@ -62,8 +62,6 @@ pub fn build_rolegroup_config_map(
     cluster_info: &KubernetesClusterInfo,
     metadata: &ObjectMetaBuilder,
     rolegroup_ref: &RoleGroupRef<v1alpha1::HdfsCluster>,
-    namenode_podrefs: &[HdfsPodRef],
-    journalnode_podrefs: &[HdfsPodRef],
 ) -> Result<ConfigMap> {
     tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref);
 
@@ -88,8 +86,6 @@ pub fn build_rolegroup_config_map(
         cluster,
         cluster_info,
         merged_config,
-        namenode_podrefs,
-        journalnode_podrefs,
         config_overrides.hdfs_site_xml.clone(),
     );
     let core_site_xml = core_site::build(

rust/operator-binary/src/controller/build/properties/hdfs_site.rs

@@ -7,10 +7,8 @@ use stackable_operator::{
 };
 
 use crate::{
-    config::HdfsSiteConfigBuilder,
-    controller::build::properties::resolved_overrides,
-    crd::{AnyNodeConfig, HdfsPodRef},
-    hdfs_controller::ValidatedCluster,
+    config::HdfsSiteConfigBuilder, controller::build::properties::resolved_overrides,
+    crd::AnyNodeConfig, hdfs_controller::ValidatedCluster,
 };
 
 /// Renders `hdfs-site.xml`: operator defaults, HA wiring derived from the pod
@@ -19,11 +17,11 @@ pub fn build(
     cluster: &ValidatedCluster,
     cluster_info: &KubernetesClusterInfo,
     merged_config: &AnyNodeConfig,
-    namenode_podrefs: &[HdfsPodRef],
-    journalnode_podrefs: &[HdfsPodRef],
     overrides: KeyValueConfigOverrides,
 ) -> String {
     let cluster_config = &cluster.cluster_config;
+    let namenode_podrefs = &cluster.namenode_podrefs;
+    let journalnode_podrefs = &cluster.journalnode_podrefs;
     // IMPORTANT: these folders must be under the volume mount point, otherwise they will not
     // be formatted by the namenode, or used by the other services.
     // See also: https://github.com/apache-spark-on-k8s/kubernetes-HDFS/commit/aef9586ecc8551ca0f0a468c3b917d8c38f494a0
@@ -133,8 +131,6 @@ mod tests {
             &validated_cluster(),
             &cluster_info(),
             &merged,
-            &[],
-            &[],
             config_overrides(&[]),
         );
         assert!(
@@ -154,8 +150,6 @@ mod tests {
             &validated_cluster(),
             &cluster_info(),
             &merged,
-            &[],
-            &[],
             config_overrides(&[("dfs.replication", "5")]),
         );
         assert!(

rust/operator-binary/src/controller/build/properties/mod.rs

@@ -94,6 +94,7 @@ apiVersion: hdfs.stackable.tech/v1alpha1
 kind: HdfsCluster
 metadata:
   name: hdfs
+  namespace: default
 spec:
   image:
     productVersion: 3.4.0

rust/operator-binary/src/controller/validate.rs

@@ -42,6 +42,9 @@ pub enum Error {
 
     #[snafu(display("failed to resolve and merge config for role and role group"))]
     FailedToResolveConfig { source: crate::crd::Error },
+
+    #[snafu(display("failed to create pod references"))]
+    CreatePodReferences { source: crate::crd::Error },
 }
 
 pub fn validate_cluster(
@@ -103,12 +106,23 @@ pub fn validate_cluster(
         role_groups.insert(hdfs_role, group_configs);
     }
 
+    // A list of all name and journal nodes across all role groups is needed for all
+    // ConfigMaps and initialization checks.
+    let namenode_podrefs = hdfs
+        .pod_refs(&HdfsNodeRole::Name)
+        .context(CreatePodReferencesSnafu)?;
+    let journalnode_podrefs = hdfs
+        .pod_refs(&HdfsNodeRole::Journal)
+        .context(CreatePodReferencesSnafu)?;
+
     Ok(ValidatedCluster {
         name: ClusterName::from_str(&hdfs.name_any()).context(InvalidClusterNameSnafu)?,
         image: resolved_product_image,
         cluster_config: ValidatedClusterConfig::resolve(hdfs, hdfs_opa_config),
         role_groups,
         role_configs,
+        namenode_podrefs,
+        journalnode_podrefs,
     })
 }
 

rust/operator-binary/src/crd/mod.rs

@@ -1161,6 +1161,7 @@ impl HdfsNodeRole {
 /// Reference to a single `Pod` that is a component of a [`HdfsCluster`]
 ///
 /// Used for service discovery.
+#[derive(Clone, Debug)]
 pub struct HdfsPodRef {
     pub namespace: String,
     pub role_group_service_name: String,

rust/operator-binary/src/hdfs_controller.rs

@@ -77,6 +77,12 @@ pub struct ValidatedCluster {
     pub cluster_config: ValidatedClusterConfig,
     pub role_groups: BTreeMap<HdfsNodeRole, BTreeMap<String, ValidatedRoleGroupConfig>>,
     pub role_configs: BTreeMap<HdfsNodeRole, ValidatedRoleConfig>,
+    /// Pod references for all namenodes across all role groups. Needed for all
+    /// ConfigMaps and initialization checks. Resolved once during validation.
+    pub namenode_podrefs: Vec<HdfsPodRef>,
+    /// Pod references for all journalnodes across all role groups. Resolved once
+    /// during validation.
+    pub journalnode_podrefs: Vec<HdfsPodRef>,
 }
 
 /// Cluster-wide settings resolved once during validation, so the build steps no
@@ -205,9 +211,6 @@ pub enum Error {
         source: stackable_operator::cluster_resources::Error,
     },
 
-    #[snafu(display("failed to create pod references"))]
-    CreatePodReferences { source: crate::crd::Error },
-
     #[snafu(display("failed to create cluster event"))]
     FailedToCreateClusterEvent { source: crate::event::Error },
 
@@ -296,13 +299,6 @@ pub async fn reconcile_hdfs(
     let resolved_product_image = &validated.image;
 
     let hdfs_obj_ref = hdfs.object_ref(&());
-    // A list of all name and journal nodes across all role groups is needed for all ConfigMaps and initialization checks.
-    let namenode_podrefs = hdfs
-        .pod_refs(&HdfsNodeRole::Name)
-        .context(CreatePodReferencesSnafu)?;
-    let journalnode_podrefs = hdfs
-        .pod_refs(&HdfsNodeRole::Journal)
-        .context(CreatePodReferencesSnafu)?;
 
     let mut cluster_resources = ClusterResources::new(
         APP_NAME,
@@ -409,8 +405,6 @@ pub async fn reconcile_hdfs(
                 &client.kubernetes_cluster_info,
                 metadata,
                 &rolegroup_ref,
-                &namenode_podrefs,
-                &journalnode_podrefs,
             )
             .context(BuildRoleGroupConfigMapSnafu)?;
 
@@ -423,7 +417,7 @@ pub async fn reconcile_hdfs(
                 resolved_product_image,
                 Some(env_overrides),
                 merged_config,
-                &namenode_podrefs,
+                &validated.namenode_podrefs,
                 &rbac_sa,
             )?;
 
@@ -686,6 +680,7 @@ apiVersion: hdfs.stackable.tech/v1alpha1
 kind: HdfsCluster
 metadata:
   name: hdfs
+  namespace: default
 spec:
   image:
     productVersion: 3.4.0