Add pause/stop feature to operator (#348)

Techassi · adwk67 · Techassi · commit 5185d677f99f · 2023-04-04T09:24:31.000Z
# Description See [#72](stackabletech/issues#72). ## Definition of Done Checklist - Not all of these items are applicable to all PRs, the author should update this template to only leave the boxes in that are relevant - Please make sure all these things are done and tick the boxes ```[tasklist] # Author - [ ] CRD changes approved - [ ] Helm chart can be installed and deployed operator works - [ ] Integration tests passed (for non trivial changes) ``` ```[tasklist] # Reviewer - [x] Code contains useful comments - [x] (Integration-)Test cases added - [ ] Documentation added or updated - [x] Changelog updated - [x] Cargo.toml only contains references to git tags (not specific commits or branches) ``` ```[tasklist] # Acceptance - [ ] Feature Tracker has been updated - [ ] Proper release label has been added ``` Once the review is done, comment `bors r+` (or `bors merge`) to merge. [Further information](https://bors.tech/documentation/getting-started/#reviewing-pull-requests) Co-authored-by: Andrew Kenworthy <andrew.kenworthy@stackable.de>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 
 - Log aggregation added ([#326]).
 - Deploy default and support custom affinities ([#337]).
+- Extend cluster resources for status and cluster operation (paused, stopped) ([#348])
 
 ### Changed
 
@@ -19,6 +20,7 @@
 [#326]: https://github.com/stackabletech/superset-operator/pull/326
 [#337]: https://github.com/stackabletech/superset-operator/pull/337
 [#344]: https://github.com/stackabletech/superset-operator/pull/344
+[#348]: https://github.com/stackabletech/superset-operator/pull/348
 
 ## [23.1.0] - 2023-01-23
 
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/deploy/helm/superset-operator/crds/crds.yaml b/deploy/helm/superset-operator/crds/crds.yaml
@@ -47,6 +47,21 @@ spec:
                       description: This role will be given in addition to any AUTH_ROLES_MAPPING. Gets mapped to `AUTH_USER_REGISTRATION_ROLE`
                       type: string
                   type: object
+                clusterOperation:
+                  default:
+                    stopped: false
+                    reconciliationPaused: false
+                  description: Cluster operations like pause reconciliation or cluster stop.
+                  properties:
+                    reconciliationPaused:
+                      default: false
+                      description: Flag to stop cluster reconciliation by the operator. This means that all changes in the custom resource spec are ignored until this flag is set to false or removed. The operator will however still watch the deployed resources at the time and update the custom resource status field. If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over `stopped` and stop the reconciliation immediately.
+                      type: boolean
+                    stopped:
+                      default: false
+                      description: Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 and therefore stopped and removed. If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation and `stopped` will take no effect until `reconciliationPaused` is set to false or removed.
+                      type: boolean
+                  type: object
                 credentialsSecret:
                   type: string
                 databaseInitialization:
diff --git a/docs/modules/superset/pages/cluster_operations.adoc b/docs/modules/superset/pages/cluster_operations.adoc
@@ -0,0 +1,4 @@
+= Cluster Operation
+
+Superset installations can be configured with different cluster operations like pausing reconciliation or stopping the
+cluster. See xref:concepts:cluster_operations.adoc[cluster operations] for more details.
diff --git a/docs/modules/superset/partials/nav.adoc b/docs/modules/superset/partials/nav.adoc
@@ -10,3 +10,4 @@
 ** xref:superset:usage-guide/logging.adoc[]
 ** xref:superset:usage-guide/configuration-environment-overrides.adoc[]
 * xref:superset:configuration.adoc[]
+* xref:superset:cluster_operations.adoc[]
diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml
@@ -11,7 +11,7 @@ publish = false
 [dependencies]
 serde = "1.0"
 serde_json = "1.0"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.36.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.39.0" }
 strum = { version = "0.24", features = ["derive"] }
 snafu = "0.7"
 tracing = "0.1"
diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs
@@ -6,6 +6,7 @@ use affinity::get_affinity;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_operator::commons::affinity::StackableAffinity;
+use stackable_operator::commons::cluster_operation::ClusterOperation;
 use stackable_operator::commons::product_image_selection::ProductImage;
 use stackable_operator::kube::ResourceExt;
 use stackable_operator::role_utils::RoleGroup;
@@ -164,6 +165,9 @@ pub struct SupersetClusterSpec {
     /// Use with caution.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub service_type: Option<ServiceType>,
+    /// Cluster operations like pause reconciliation or cluster stop.
+    #[serde(default)]
+    pub cluster_operation: ClusterOperation,
 }
 
 // TODO: Temporary solution until listener-operator is finished
diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml
@@ -16,13 +16,13 @@ futures = { version = "0.3", features = ["compat"] }
 indoc = "1.0"
 serde = "1.0"
 snafu = "0.7"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.36.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.39.0" }
 stackable-superset-crd = { path = "../crd" }
 strum = { version = "0.24", features = ["derive"] }
 tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] }
 tracing = "0.1"
 
 [build-dependencies]
 built = { version =  "0.5", features = ["chrono", "git2"] }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.36.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.39.0" }
 stackable-superset-crd = { path = "../crd" }
diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs
@@ -12,6 +12,7 @@ use crate::{
 
 use indoc::formatdoc;
 use snafu::{OptionExt, ResultExt, Snafu};
+use stackable_operator::cluster_resources::ClusterResourceApplyStrategy;
 use stackable_operator::commons::product_image_selection::ResolvedProductImage;
 use stackable_operator::{
     builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder},
@@ -268,12 +269,13 @@ pub async fn reconcile_superset(superset: Arc<SupersetCluster>, ctx: Arc<Ctx>) -
         OPERATOR_NAME,
         SUPERSET_CONTROLLER_NAME,
         &superset.object_ref(&()),
+        ClusterResourceApplyStrategy::from(&superset.spec.cluster_operation),
     )
     .context(CreateClusterResourcesSnafu)?;
 
     let node_role_service = build_node_role_service(&superset, &resolved_product_image)?;
     cluster_resources
-        .add(client, &node_role_service)
+        .add(client, node_role_service)
         .await
         .context(ApplyRoleServiceSnafu)?;
 
@@ -325,19 +327,19 @@ pub async fn reconcile_superset(superset: Arc<SupersetCluster>, ctx: Arc<Ctx>) -
             &config,
         )?;
         cluster_resources
-            .add(client, &rg_service)
+            .add(client, rg_service)
             .await
             .with_context(|_| ApplyRoleGroupServiceSnafu {
                 rolegroup: rolegroup.clone(),
             })?;
         cluster_resources
-            .add(client, &rg_configmap)
+            .add(client, rg_configmap)
             .await
             .with_context(|_| ApplyRoleGroupConfigSnafu {
                 rolegroup: rolegroup.clone(),
             })?;
         cluster_resources
-            .add(client, &rg_statefulset)
+            .add(client, rg_statefulset)
             .await
             .with_context(|_| ApplyRoleGroupStatefulSetSnafu {
                 rolegroup: rolegroup.clone(),
diff --git a/tests/templates/kuttl/cluster-operation/00-assert.yaml b/tests/templates/kuttl/cluster-operation/00-assert.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: test-superset-postgresql
+timeout: 480
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: superset-postgresql
+status:
+  readyReplicas: 1
+  replicas: 1
diff --git a/tests/templates/kuttl/cluster-operation/00-install-postgresql.yaml b/tests/templates/kuttl/cluster-operation/00-install-postgresql.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: >-
+      helm install superset-postgresql
+      --namespace $NAMESPACE
+      --version 12.1.5
+      --set auth.username=superset
+      --set auth.password=superset
+      --set auth.database=superset
+      --repo https://charts.bitnami.com/bitnami postgresql
diff --git a/tests/templates/kuttl/cluster-operation/01-assert.yaml.j2 b/tests/templates/kuttl/cluster-operation/01-assert.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}
diff --git a/tests/templates/kuttl/cluster-operation/01-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/cluster-operation/01-install-vector-aggregator-discovery-configmap.yaml.j2
@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}
diff --git a/tests/templates/kuttl/cluster-operation/02-assert.yaml b/tests/templates/kuttl/cluster-operation/02-assert.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: superset-node-default
+status:
+  readyReplicas: 1
+  replicas: 1
diff --git a/tests/templates/kuttl/cluster-operation/02-install-superset.yaml.j2 b/tests/templates/kuttl/cluster-operation/02-install-superset.yaml.j2
@@ -0,0 +1,41 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: superset-credentials
+type: Opaque
+stringData:
+  adminUser.username: admin
+  adminUser.firstname: Superset
+  adminUser.lastname: Admin
+  adminUser.email: admin@superset.com
+  adminUser.password: admin
+  connections.secretKey: thisISaSECRET_1234
+  connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset
+---
+apiVersion: superset.stackable.tech/v1alpha1
+kind: SupersetCluster
+metadata:
+  name: superset
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['superset-latest'].split('-stackable')[0] }}"
+    stackableVersion: "{{ test_scenario['values']['superset-latest'].split('-stackable')[1] }}"
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  credentialsSecret: superset-credentials
+  loadExamplesOnInit: false
+  nodes:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1
diff --git a/tests/templates/kuttl/cluster-operation/03-assert.yaml b/tests/templates/kuttl/cluster-operation/03-assert.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: superset-node-default
+status:
+  replicas: 1
diff --git a/tests/templates/kuttl/cluster-operation/03-install-superset-pause.yaml.j2 b/tests/templates/kuttl/cluster-operation/03-install-superset-pause.yaml.j2
@@ -0,0 +1,44 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: superset-credentials
+type: Opaque
+stringData:
+  adminUser.username: admin
+  adminUser.firstname: Superset
+  adminUser.lastname: Admin
+  adminUser.email: admin@superset.com
+  adminUser.password: admin
+  connections.secretKey: thisISaSECRET_1234
+  connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset
+---
+apiVersion: superset.stackable.tech/v1alpha1
+kind: SupersetCluster
+metadata:
+  name: superset
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['superset-latest'].split('-stackable')[0] }}"
+    stackableVersion: "{{ test_scenario['values']['superset-latest'].split('-stackable')[1] }}"
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  clusterOperation:
+    stopped: false
+    reconciliationPaused: true
+  credentialsSecret: superset-credentials
+  loadExamplesOnInit: false
+  nodes:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 2
diff --git a/tests/templates/kuttl/cluster-operation/04-assert.yaml b/tests/templates/kuttl/cluster-operation/04-assert.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: superset-node-default
+status:
+  replicas: 0
diff --git a/tests/templates/kuttl/cluster-operation/04-install-superset-stop.yaml.j2 b/tests/templates/kuttl/cluster-operation/04-install-superset-stop.yaml.j2
@@ -0,0 +1,44 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: superset-credentials
+type: Opaque
+stringData:
+  adminUser.username: admin
+  adminUser.firstname: Superset
+  adminUser.lastname: Admin
+  adminUser.email: admin@superset.com
+  adminUser.password: admin
+  connections.secretKey: thisISaSECRET_1234
+  connections.sqlalchemyDatabaseUri: postgresql://superset:superset@superset-postgresql/superset
+---
+apiVersion: superset.stackable.tech/v1alpha1
+kind: SupersetCluster
+metadata:
+  name: superset
+spec:
+  image:
+    productVersion: "{{ test_scenario['values']['superset-latest'].split('-stackable')[0] }}"
+    stackableVersion: "{{ test_scenario['values']['superset-latest'].split('-stackable')[1] }}"
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  clusterOperation:
+    stopped: true
+    reconciliationPaused: false
+  credentialsSecret: superset-credentials
+  loadExamplesOnInit: false
+  nodes:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 2
diff --git a/tests/templates/kuttl/cluster-operation/05-assert.yaml b/tests/templates/kuttl/cluster-operation/05-assert.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: install-superset
+timeout: 300
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: superset-node-default
+status:
+  replicas: 2
diff --git a/tests/templates/kuttl/cluster-operation/05-install-superset-restart.yaml.j2 b/tests/templates/kuttl/cluster-operation/05-install-superset-restart.yaml.j2
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
