feat: Add annotations to the StatefulSet to ignore resources with security settings

Author: siegfriedweber

rust/operator-binary/src/controller/build/role_builder.rs

@@ -13,6 +13,7 @@ use stackable_operator::{
             rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject},
         },
     },
+    kube::api::ObjectMeta,
     kvp::{
         Label, Labels,
         consts::{STACKABLE_VENDOR_KEY, STACKABLE_VENDOR_VALUE},
@@ -28,8 +29,7 @@ use crate::{
     framework::{
         NameIsValidLabelValue,
         builder::{
-            meta::{annotation_ignore_restarter, ownerreference_from_resource},
-            pdb::pod_disruption_budget_builder_with_role,
+            meta::ownerreference_from_resource, pdb::pod_disruption_budget_builder_with_role,
         },
         role_utils::ResourceNames,
         types::{
@@ -81,9 +81,7 @@ impl<'a> RoleBuilder<'a> {
 
     /// Builds a ServiceAccount used by all role-groups
     pub fn build_service_account(&self) -> ServiceAccount {
-        let metadata = self
-            .common_metadata(self.resource_names.service_account_name())
-            .build();
+        let metadata = self.common_metadata(self.resource_names.service_account_name());
 
         ServiceAccount {
             metadata,
@@ -93,9 +91,7 @@ impl<'a> RoleBuilder<'a> {
 
     /// Builds a RoleBinding used by all role-groups
     pub fn build_role_binding(&self) -> RoleBinding {
-        let metadata = self
-            .common_metadata(self.resource_names.role_binding_name())
-            .build();
+        let metadata = self.common_metadata(self.resource_names.role_binding_name());
 
         RoleBinding {
             metadata,
@@ -121,9 +117,7 @@ impl<'a> RoleBuilder<'a> {
             ..ServicePort::default()
         }];
 
-        let metadata = self
-            .common_metadata(seed_nodes_service_name(&self.cluster.name))
-            .build();
+        let metadata = self.common_metadata(seed_nodes_service_name(&self.cluster.name));
 
         let service_selector =
             RoleGroupBuilder::cluster_manager_labels(&self.cluster, self.context_names);
@@ -147,9 +141,7 @@ impl<'a> RoleBuilder<'a> {
 
     /// Builds a Listener whose status is used to populate the discovery ConfigMap.
     pub fn build_discovery_service_listener(&self) -> listener::v1alpha1::Listener {
-        let metadata = self
-            .common_metadata(discovery_service_listener_name(&self.cluster.name))
-            .build();
+        let metadata = self.common_metadata(discovery_service_listener_name(&self.cluster.name));
 
         let listener_class = &self.cluster.role_config.discovery_service_listener_class;
 
@@ -178,9 +170,7 @@ impl<'a> RoleBuilder<'a> {
     pub fn build_maybe_discovery_config_map(&self) -> Option<ConfigMap> {
         let discovery_endpoint = self.cluster.discovery_endpoint.as_ref()?;
 
-        let metadata = self
-            .common_metadata(discovery_config_map_name(&self.cluster.name))
-            .build();
+        let metadata = self.common_metadata(discovery_config_map_name(&self.cluster.name));
 
         let protocol = if self.cluster.is_server_tls_enabled() {
             "https"
@@ -221,10 +211,7 @@ impl<'a> RoleBuilder<'a> {
     /// Returns `None` if the security plugin is disabled or all configuration files are
     /// references.
     pub fn build_maybe_security_config_map(&self) -> Option<ConfigMap> {
-        let metadata = self
-            .common_metadata(security_config_map_name(&self.cluster.name))
-            .with_annotation(annotation_ignore_restarter())
-            .build();
+        let metadata = self.common_metadata(security_config_map_name(&self.cluster.name));
 
         let mut data = BTreeMap::new();
 
@@ -277,20 +264,17 @@ impl<'a> RoleBuilder<'a> {
     }
 
     /// Common metadata for role resources
-    fn common_metadata(&self, resource_name: impl Into<String>) -> ObjectMetaBuilder {
-        let mut builder = ObjectMetaBuilder::new();
-
-        builder
+    fn common_metadata(&self, resource_name: impl Into<String>) -> ObjectMeta {
+        ObjectMetaBuilder::new()
             .name(resource_name)
             .namespace(&self.cluster.namespace)
             .ownerreference(ownerreference_from_resource(
                 &self.cluster,
                 None,
                 Some(true),
             ))
-            .with_labels(self.labels());
-
-        builder
+            .with_labels(self.labels())
+            .build()
     }
 
     /// Common labels for role resources
@@ -763,9 +747,6 @@ mod tests {
                 "apiVersion": "v1",
                 "kind": "ConfigMap",
                 "metadata": {
-                    "annotations": {
-                        "restarter.stackable.tech/ignore": "true"
-                    },
                     "labels": {
                         "app.kubernetes.io/component": "nodes",
                         "app.kubernetes.io/instance": "my-opensearch-cluster",

rust/operator-binary/src/controller/build/role_group_builder.rs

@@ -1,6 +1,9 @@
 //! Builder for role group resources
 
-use std::{collections::BTreeMap, str::FromStr};
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    str::FromStr,
+};
 
 use stackable_operator::{
     builder::{
@@ -70,8 +73,8 @@ use crate::{
         role_group_utils::ResourceNames,
         types::{
             kubernetes::{
-                ListenerName, PersistentVolumeClaimName, SecretClassName, ServiceAccountName,
-                ServiceName, VolumeName,
+                ConfigMapName, ListenerName, PersistentVolumeClaimName, SecretClassName,
+                SecretName, ServiceAccountName, ServiceName, VolumeName,
             },
             operator::RoleGroupName,
         },
@@ -309,6 +312,7 @@ impl<'a> RoleGroupBuilder<'a> {
         let metadata = self
             .common_metadata(self.resource_names.stateful_set_name())
             .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned())
+            .with_annotations(self.restarter_ignore_annotations())
             .build();
 
         let template = self.build_pod_template();
@@ -373,6 +377,42 @@ impl<'a> RoleGroupBuilder<'a> {
         }
     }
 
+    fn restarter_ignore_annotations(&self) -> Annotations {
+        let (security_settings_config_maps, security_settings_secrets) =
+            self.security_settings_resource_names();
+
+        let restarter_ignore_config_maps_annotations = security_settings_config_maps
+            .iter()
+            .enumerate()
+            .map(|(i, config_map_name)| {
+                (
+                    format!("restarter.stackable.tech/ignore-config-map.{i}"),
+                    config_map_name.to_string(),
+                )
+            });
+        let restarter_ignore_secrets_annotations = security_settings_secrets
+            .iter()
+            .enumerate()
+            .map(|(i, secret_name)| {
+                (
+                    format!("restarter.stackable.tech/ignore-secret.{i}"),
+                    secret_name.to_string(),
+                )
+            });
+
+        // The expectation is tested in the unit tests.
+        Annotations::try_from(
+            restarter_ignore_config_maps_annotations
+                .chain(restarter_ignore_secrets_annotations)
+                .collect::<BTreeMap<_, _>>(),
+        )
+        .expect(
+            "should contain only valid annotations because the annotation keys are statically \
+            defined apart from the index number and the names of ConfigMaps and Secrets are valid \
+            annotation values.",
+        )
+    }
+
     /// Builds the [`PodTemplateSpec`] for the [`StatefulSet`] of the role group
     fn build_pod_template(&self) -> PodTemplateSpec {
         let mut node_role_labels = Labels::new();
@@ -1123,6 +1163,39 @@ impl<'a> RoleGroupBuilder<'a> {
         }]
     }
 
+    fn security_settings_resource_names(&self) -> (BTreeSet<ConfigMapName>, BTreeSet<SecretName>) {
+        let mut config_map_names = BTreeSet::new();
+        let mut secret_names = BTreeSet::new();
+
+        if let RoleGroupSecurityMode::Initializing { settings, .. }
+        | RoleGroupSecurityMode::Managing { settings, .. } = &self.security_mode
+        {
+            for file_type in settings {
+                match &file_type.content {
+                    v1alpha1::SecuritySettingsFileTypeContent::Value(_) => {
+                        config_map_names.insert(security_config_map_name(&self.cluster.name));
+                    }
+                    v1alpha1::SecuritySettingsFileTypeContent::ValueFrom(
+                        v1alpha1::SecuritySettingsFileTypeContentValueFrom::ConfigMapKeyRef(
+                            v1alpha1::ConfigMapKeyRef { name, .. },
+                        ),
+                    ) => {
+                        config_map_names.insert(name.clone());
+                    }
+                    v1alpha1::SecuritySettingsFileTypeContent::ValueFrom(
+                        v1alpha1::SecuritySettingsFileTypeContentValueFrom::SecretKeyRef(
+                            v1alpha1::SecretKeyRef { name, .. },
+                        ),
+                    ) => {
+                        secret_names.insert(name.clone());
+                    }
+                };
+            }
+        }
+
+        (config_map_names, secret_names)
+    }
+
     /// Builds the security settings volumes for the [`PodTemplateSpec`]
     /// It is not checked if these volumes are required in this role group.
     fn build_security_settings_volumes(
@@ -1712,6 +1785,15 @@ mod tests {
         let stateful_set = serde_json::to_value(role_group_builder.build_stateful_set())
             .expect("should be serializable");
 
+        let expected_annotations = match security_mode {
+            TestSecurityMode::Initializing | TestSecurityMode::Managing => json!({
+                "restarter.stackable.tech/ignore-config-map.0": "my-opensearch-cluster-security-config",
+                "restarter.stackable.tech/ignore-config-map.1": "opensearch-security-config",
+                "restarter.stackable.tech/ignore-secret.0": "opensearch-security-config",
+            }),
+            TestSecurityMode::Disabled | TestSecurityMode::Participating => json!({}),
+        };
+
         let expected_opensearch_container_volume_mounts = match security_mode {
             TestSecurityMode::Initializing => json!([
                 {
@@ -3026,6 +3108,7 @@ mod tests {
                 "apiVersion": "apps/v1",
                 "kind": "StatefulSet",
                 "metadata": {
+                    "annotations": expected_annotations,
                     "labels": {
                         "app.kubernetes.io/component": "nodes",
                         "app.kubernetes.io/instance": "my-opensearch-cluster",

rust/operator-binary/src/framework/builder/meta.rs

@@ -1,7 +1,6 @@
 use stackable_operator::{
     builder::meta::OwnerReferenceBuilder,
     k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference, kube::Resource,
-    kvp::Annotation,
 };
 
 use crate::framework::{HasName, HasUid};
@@ -29,15 +28,6 @@ pub fn ownerreference_from_resource(
         )
 }
 
-/// Annotation which signals the restarter to ignore this resource.
-pub fn annotation_ignore_restarter() -> Annotation {
-    Annotation::try_from((
-        "restarter.stackable.tech/ignore".to_owned(),
-        "true".to_owned(),
-    ))
-    .expect("should be a valid annotation")
-}
-
 #[cfg(test)]
 mod tests {
     use std::borrow::Cow;
@@ -47,10 +37,7 @@ mod tests {
         kube::Resource,
     };
 
-    use crate::framework::{
-        HasName, HasUid, Uid,
-        builder::meta::{annotation_ignore_restarter, ownerreference_from_resource},
-    };
+    use crate::framework::{HasName, HasUid, Uid, builder::meta::ownerreference_from_resource};
 
     struct Cluster {
         object_meta: ObjectMeta,
@@ -134,10 +121,4 @@ mod tests {
 
         assert_eq!(expected_owner_reference, actual_owner_reference);
     }
-
-    #[test]
-    fn test_annotation_ignore_restarter() {
-        // Test that the function does not panic
-        annotation_ignore_restarter();
-    }
 }

tests/templates/kuttl/security-config/10-security-config.yaml

@@ -3,8 +3,6 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: security-config-file-internal-users
-  annotations:
-    restarter.stackable.tech/ignore: "true"
 stringData:
   internal_users.yml: |
     ---
@@ -23,8 +21,6 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: security-config
-  annotations:
-    restarter.stackable.tech/ignore: "true"
 data:
   roles.yml: |
     ---

tests/templates/kuttl/security-config/11-assert.yaml

@@ -23,6 +23,8 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: opensearch-nodes-security-config
+  annotations:
+    restarter.stackable.tech/ignore-config-map.0: "opensearch-security-config"
 status:
   readyReplicas: 1
   replicas: 1
@@ -31,8 +33,6 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: opensearch-security-config
-  annotations:
-    restarter.stackable.tech/ignore: "true"
 data:
   action_groups.yml: '{"_meta":{"config_version":2,"type":"actiongroups"}}'
   allowlist.yml: '{"_meta":{"config_version":2,"type":"allowlist"},"config":{"enabled":false}}'