refactor: extract dereference and validate steps into controller modules (#783)

* refactor: extract dereference and validate steps into controller modules

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* changelog

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* formatting

* extend assert step to increase test coverage by checking most generated resources

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* added resources assert and re-org as per hive review feedback

* extend STS definition for the assert and remove duplicate envOverrides test

* Expand smoke test assertions with full STS specs and ConfigMap snapshots

  30-assert now includes args, imagePullPolicy, defaultMode, and all
  container specs for the three HDFS StatefulSets. 31-assert is refactored
  to diff ConfigMap data against external snapshot files using envsubst and
  yq normalisation, replacing the previous inline envOverrides checks.

  Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* linting

* more linting

* Use jq to normalise trailing newlines in ConfigMap snapshot diffs

  Replaces variable-based comparison with pipe-to-file approach. Both
  sides are normalised via jq (collapsing trailing \n+ to \n) so YAML
  block scalar style differences (|, |+) don't cause false failures.
  Temp files use $NAMESPACE prefix for concurrent test safety.

  Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: adwk67

CHANGELOG.md

@@ -16,10 +16,12 @@ All notable changes to this project will be documented in this file.
   `security.properties`).
   Previously, arbitrary file names were silently accepted and ignored ([#777]).
 - Bump `stackable-operator` to 0.111.1 ([#777], [#778]).
+- Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#783]).
 
 [#770]: https://github.com/stackabletech/hdfs-operator/pull/770
 [#777]: https://github.com/stackabletech/hdfs-operator/pull/777
 [#778]: https://github.com/stackabletech/hdfs-operator/pull/778
+[#783]: https://github.com/stackabletech/hdfs-operator/pull/783
 
 ## [26.3.0] - 2026-03-16
 

Cargo.nix

@@ -0,0 +1,30 @@
+use snafu::{ResultExt, Snafu};
+
+use crate::{crd::v1alpha1, security::opa::HdfsOpaConfig};
+
+#[derive(Snafu, Debug)]
+pub enum Error {
+    #[snafu(display("invalid OPA configuration"))]
+    InvalidOpaConfig { source: crate::security::opa::Error },
+}
+
+/// External references resolved during the dereference step.
+pub struct DereferencedObjects {
+    pub hdfs_opa_config: Option<HdfsOpaConfig>,
+}
+
+pub async fn dereference(
+    client: &stackable_operator::client::Client,
+    hdfs: &v1alpha1::HdfsCluster,
+) -> Result<DereferencedObjects, Error> {
+    let hdfs_opa_config = match &hdfs.spec.cluster_config.authorization {
+        Some(opa_config) => Some(
+            HdfsOpaConfig::from_opa_config(client, hdfs, opa_config)
+                .await
+                .context(InvalidOpaConfigSnafu)?,
+        ),
+        None => None,
+    };
+
+    Ok(DereferencedObjects { hdfs_opa_config })
+}

crate-hashes.json

@@ -0,0 +1,2 @@
+pub mod dereference;
+pub mod validate;

rust/operator-binary/src/controller/dereference.rs

@@ -0,0 +1,115 @@
+use std::{collections::BTreeMap, str::FromStr};
+
+use product_config::ProductConfigManager;
+use snafu::{ResultExt, Snafu};
+use stackable_operator::{
+    commons::product_image_selection,
+    product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config},
+    role_utils::GenericRoleConfig,
+};
+
+use crate::{
+    crd::{HdfsNodeRole, v1alpha1},
+    hdfs_controller::{
+        CONTAINER_IMAGE_BASE_NAME, ValidatedCluster, ValidatedRoleConfig, ValidatedRoleGroupConfig,
+    },
+    security::opa::HdfsOpaConfig,
+};
+
+#[derive(Snafu, Debug)]
+pub enum Error {
+    #[snafu(display("failed to resolve product image"))]
+    ResolveProductImage {
+        source: product_image_selection::Error,
+    },
+
+    #[snafu(display("invalid role properties"))]
+    RoleProperties { source: crate::crd::Error },
+
+    #[snafu(display("failed to generate product config"))]
+    GenerateProductConfig {
+        source: stackable_operator::product_config_utils::Error,
+    },
+
+    #[snafu(display("invalid product configuration"))]
+    InvalidProductConfig {
+        source: stackable_operator::product_config_utils::Error,
+    },
+
+    #[snafu(display("could not parse HDFS role [{role}]"))]
+    UnidentifiedHdfsRole {
+        source: strum::ParseError,
+        role: String,
+    },
+
+    #[snafu(display("failed to resolve and merge config for role and role group"))]
+    FailedToResolveConfig { source: crate::crd::Error },
+}
+
+pub fn validate_cluster(
+    hdfs: &v1alpha1::HdfsCluster,
+    image_repository: &str,
+    product_config_manager: &ProductConfigManager,
+    hdfs_opa_config: Option<HdfsOpaConfig>,
+) -> Result<ValidatedCluster, Error> {
+    let resolved_product_image = hdfs
+        .spec
+        .image
+        .resolve(
+            CONTAINER_IMAGE_BASE_NAME,
+            image_repository,
+            crate::built_info::PKG_VERSION,
+        )
+        .context(ResolveProductImageSnafu)?;
+
+    let roles = hdfs.build_role_properties().context(RolePropertiesSnafu)?;
+
+    let validated_config = validate_all_roles_and_groups_config(
+        &resolved_product_image.product_version,
+        &transform_all_roles_to_config(hdfs, &roles).context(GenerateProductConfigSnafu)?,
+        product_config_manager,
+        false,
+        false,
+    )
+    .context(InvalidProductConfigSnafu)?;
+
+    let mut role_groups = BTreeMap::new();
+    let mut role_configs = BTreeMap::new();
+
+    for (role_name, group_config) in validated_config.iter() {
+        let hdfs_role = HdfsNodeRole::from_str(role_name).context(UnidentifiedHdfsRoleSnafu {
+            role: role_name.to_string(),
+        })?;
+
+        if let Some(GenericRoleConfig {
+            pod_disruption_budget: pdb,
+        }) = hdfs.role_config(&hdfs_role)
+        {
+            role_configs.insert(hdfs_role, ValidatedRoleConfig { pdb: pdb.clone() });
+        }
+
+        let mut group_configs = BTreeMap::new();
+        for (rolegroup_name, rolegroup_config) in group_config.iter() {
+            let merged_config = hdfs_role
+                .merged_config(hdfs, rolegroup_name)
+                .context(FailedToResolveConfigSnafu)?;
+
+            group_configs.insert(
+                rolegroup_name.clone(),
+                ValidatedRoleGroupConfig {
+                    merged_config,
+                    product_config_properties: rolegroup_config.clone(),
+                },
+            );
+        }
+
+        role_groups.insert(hdfs_role, group_configs);
+    }
+
+    Ok(ValidatedCluster {
+        image: resolved_product_image,
+        role_groups,
+        role_configs,
+        hdfs_opa_config,
+    })
+}

rust/operator-binary/src/controller/mod.rs

@@ -993,7 +993,7 @@ pub struct CommonNodeConfig {
 }
 
 /// Configuration for a rolegroup of an unknown type.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub enum AnyNodeConfig {
     Name(NameNodeConfig),
     Data(DataNodeConfig),
@@ -1087,7 +1087,9 @@ impl AnyNodeConfig {
     Eq,
     Hash,
     JsonSchema,
+    Ord,
     PartialEq,
+    PartialOrd,
     Serialize,
 )]
 pub enum HdfsNodeRole {