feat: Improve startup times by setting flow.election.max.candidates

Author: sbernauer

CHANGELOG.md

@@ -18,6 +18,8 @@ All notable changes to this project will be documented in this file.
 - Bump `stackable-operator` to 0.110.1 and `kube` to 3.1.0 ([#921]).
 - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#935]).
 - Default `nifi.cluster.flow.election.max.wait.time` to NiFi's upstream value (`5 mins`) instead of the operator's previous `1 mins`. The operator no longer sets this property explicitly; the previous shorter value was left over from a TODO marked as "for testing" and may have caused flow election to settle on incomplete vote sets in cold-start scenarios ([#936]).
+- In case the user specifies a fixed number of NiFi nodes (i.e. no auto-scaling), set `nifi.cluster.flow.election.max.candidates` to that number.
+  This results in much faster NiFi startups, as it doesn't need to wait for the 5 minutes of `nifi.cluster.flow.election.max.wait.time` ([#XXX]).
 - Set `nifi.content.repository.archive.max.retention.period` to `3 days` (previously empty, which NiFi interprets as `Long.MAX_VALUE` and effectively disables time-based archive purge). Without a time-based ceiling, the content archive can grow to half the content PVC and accumulate millions of files, which makes the synchronous startup directory scan in `FileSystemRepository.initializeRepository` very slow. Users requiring a longer content-replay window can extend via `configOverrides`. The provenance audit trail is independent of this setting and unaffected ([#936]).
 - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#940]).
 
@@ -34,6 +36,7 @@ All notable changes to this project will be documented in this file.
 [#935]: https://github.com/stackabletech/nifi-operator/pull/935
 [#936]: https://github.com/stackabletech/nifi-operator/pull/936
 [#940]: https://github.com/stackabletech/nifi-operator/pull/940
+[#XXX]: https://github.com/stackabletech/nifi-operator/pull/XXX
 
 ## [26.3.0] - 2026-03-16
 

docs/modules/nifi/assets/attachments/entraid-nifi.yaml

@@ -204,7 +204,6 @@ spec:
         nifi.security.user.oidc.additional.scopes: "profile"
         nifi.security.user.oidc.claim.identifying.user: "upn"
         nifi.process.group.root.placeholder: "root"
-        nifi.cluster.flow.election.max.wait.time: "10 secs"
     podOverrides:
       spec:
         initContainers:

docs/modules/nifi/pages/usage_guide/overrides.adoc

@@ -71,7 +71,6 @@ For example per role group:
 nodes:
   roleGroups:
     default:
-      config: {}
       replicas: 1
       envOverrides:
         MY_ENV_VAR: "MY_VALUE"
@@ -86,7 +85,6 @@ nodes:
     MY_ENV_VAR: "MY_VALUE"
   roleGroups:
     default:
-      config: {}
       replicas: 1
 ----
 

rust/operator-binary/src/config/mod.rs

@@ -151,14 +151,15 @@ pub fn build_bootstrap_conf(
 
 /// Create the NiFi nifi.properties
 pub fn build_nifi_properties(
-    spec: &v1alpha1::NifiClusterSpec,
+    nifi: &v1alpha1::NifiCluster,
     resource_config: &Resources<NifiStorageConfig>,
     proxy_hosts: &str,
     auth_config: &NifiAuthenticationConfig,
     overrides: BTreeMap<String, String>,
     product_version: &str,
     git_sync_resources: &git_sync::v1alpha2::GitSyncResources,
 ) -> Result<String, Error> {
+    let cluster_config = &nifi.spec.cluster_config;
     // TODO: Remove once we dropped support for all NiFi 1.x versions
     let is_nifi_1 = product_version.starts_with("1.");
 
@@ -278,7 +279,7 @@ pub fn build_nifi_properties(
     // The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
     properties.insert(
         "nifi.state.management.provider.cluster".to_string(),
-        match spec.cluster_config.clustering_backend {
+        match cluster_config.clustering_backend {
             v1alpha1::NifiClusteringBackend::ZooKeeper { .. } => "zk-provider".to_string(),
             v1alpha1::NifiClusteringBackend::Kubernetes { .. } => "kubernetes-provider".to_string(),
         },
@@ -528,15 +529,14 @@ pub fn build_nifi_properties(
         "".to_string(),
     );
 
-    let sensitive_properties_algorithm = &spec
-        .cluster_config
+    let sensitive_properties_algorithm = cluster_config
         .sensitive_properties
         .algorithm
         .clone()
         .unwrap_or_default();
 
     sensitive_properties_algorithm
-        .check_for_nifi_version(spec.image.product_version())
+        .check_for_nifi_version(nifi.spec.image.product_version())
         .context(ConfigureSensitivePropertiesSnafu)?;
 
     properties.insert(
@@ -609,12 +609,21 @@ pub fn build_nifi_properties(
         "nifi.cluster.node.protocol.port".to_string(),
         PROTOCOL_PORT.to_string(),
     );
+
+    // In case the number of NiFi nodes is hard-coded to a fixed (no auto-scaling), we can tell
+    // this NiFi, so that startup is much quicker.
+    // Note: In case we don't know the replica count, we set it to "" as that's what we always did
+    // before.
+    let max_candidates = nifi
+        .maybe_fixed_node_count()
+        .map(|count| count.to_string())
+        .unwrap_or_default();
     properties.insert(
         "nifi.cluster.flow.election.max.candidates".to_string(),
-        "".to_string(),
+        max_candidates,
     );
 
-    match spec.cluster_config.clustering_backend {
+    match cluster_config.clustering_backend {
         v1alpha1::NifiClusteringBackend::ZooKeeper { .. } => {
             properties.insert(
                 "nifi.cluster.leader.election.implementation".to_string(),

rust/operator-binary/src/controller.rs

@@ -756,7 +756,7 @@ async fn build_node_rolegroup_config_map(
         .add_data(
             NIFI_PROPERTIES,
             build_nifi_properties(
-                &nifi.spec,
+                nifi,
                 &merged_config.resources,
                 proxy_hosts,
                 authentication_config,

rust/operator-binary/src/crd/mod.rs

@@ -289,6 +289,21 @@ impl v1alpha1::NifiCluster {
         tracing::debug!("Merged config: {:?}", conf_rolegroup);
         fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu)
     }
+
+    /// Returns [`Some<u32>`] in case the number of nodes is hard-coded to a certain value.
+    ///
+    /// This is the case when all `replicas` are set to [`Some<u16>`], in which case they are simply
+    /// summed.
+    pub fn maybe_fixed_node_count(&self) -> Option<u32> {
+        self.spec
+            .nodes
+            .as_ref()?
+            .role_groups
+            .values()
+            .map(|rg| rg.replicas.map(u32::from))
+            // As the individual replicas are [`u16`]s, this has plenty space
+            .sum()
+    }
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]

tests/templates/kuttl/cluster-sync-cs-bug/20-install-nifi.yaml.j2

@@ -186,8 +186,6 @@ spec:
       nifi.properties:
         nifi.web.https.sni.required: "false"
         nifi.web.https.sni.host.check: "false"
-        # Quicker startup, and we only have a single node
-        nifi.cluster.flow.election.max.wait.time: "10 secs"
     podOverrides:
       spec:
         initContainers:

tests/templates/kuttl/cluster_operation/20-install-nifi.yaml.j2

@@ -50,10 +50,6 @@ spec:
       gracefulShutdownTimeout: 1m
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-    configOverrides:
-      "nifi.properties":
-        # Quicker startup, and we only have a single node
-        "nifi.cluster.flow.election.max.wait.time": "10 secs"
     roleGroups:
       default:
         replicas: 2

tests/templates/kuttl/custom-components-git-sync/30_install-nifi.yaml.j2

@@ -67,10 +67,6 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-    configOverrides:
-      "nifi.properties":
-        # Quicker startup, and we only have a single node
-        "nifi.cluster.flow.election.max.wait.time": "10 secs"
     roleConfig:
       listenerClass: external-unstable
     podOverrides:

tests/templates/kuttl/external-access/30_nifi.yaml.j2

@@ -53,10 +53,6 @@ spec:
     config:
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-    configOverrides:
-      "nifi.properties":
-        # Quicker startup, and we only have a single node
-        "nifi.cluster.flow.election.max.wait.time": "10 secs"
     roleConfig:
       listenerClass: test-external-unstable-$NAMESPACE
     roleGroups: