zookeeper: add default PodAntiAffinity toggle in CR

Add a CR parameter that gives a deployer the ability to set a simple
PodAntiAffinity policy on the zookeeper replicas, ensuring the pods
run on distinct nodes. It is disabled by default.

Change-Id: I93b1987759184951dc84222f77c04777fe46f8f6

Author: mhuin

api/v1/softwarefactory_types.go

@@ -498,6 +498,11 @@ type ZookeeperSpec struct {
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default={"memory": "2Gi", "cpu": "500m"}
 	Limits *LimitsSpec `json:"limits"`
+	// Enable node anti-affinity, defaults to false so as not to fail on minikube deployments. This should
+	// be set to true on production deployments with at least 3 nodes available.
+	// +kubebuilder:default:=false
+	// +optional
+	NodeAntiAffinityEnabled *bool `json:"nodeAntiAffinityEnabled,omitempty"`
 }
 
 type CodesearchSpec struct {

api/v1/zz_generated.deepcopy.go

@@ -510,6 +510,12 @@ spec:
                     - cpu
                     - memory
                     type: object
+                  nodeAntiAffinityEnabled:
+                    default: false
+                    description: |-
+                      Enable node anti-affinity, defaults to false so as not to fail on minikube deployments. This should
+                      be set to true on production deployments with at least 3 nodes available.
+                    type: boolean
                   storage:
                     properties:
                       className:

config/crd/bases/sf.softwarefactory-project.io_softwarefactories.yaml

@@ -256,6 +256,41 @@ func (r *SFController) DeployZookeeper() bool {
 
 	zk.Spec.Template.Spec.HostAliases = base.CreateHostAliases(r.cr.Spec.HostAliases)
 
+	// Node Anti-Affinity
+	//
+	// TODO: changes to this should appear in the dry-run/verbose output, and respect user-defined affinities
+	//
+	// Also note that to disable this after it's been enabled (but why would you ?) you will have to
+	// 1. remove the setting in the SF CR
+	// 2. manually edit the zookeeper statefulset to remove the PodAntiAffinity setting
+	// This is because we do not want to interfere with custom-set rules that may be set OoB.
+	aaEnabled := false
+	if r.cr.Spec.Zookeeper.NodeAntiAffinityEnabled != nil {
+		aaEnabled = *r.cr.Spec.Zookeeper.NodeAntiAffinityEnabled
+	}
+	if aaEnabled {
+		zkAALabelSelector := metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"app": "sf",
+				"run": ZookeeperIdent,
+			},
+		}
+		zkRequiredDuringSchedulingIgnoredDuringExecution := []apiv1.PodAffinityTerm{
+			{
+				LabelSelector: &zkAALabelSelector,
+				TopologyKey:   "kubernetes.io/hostname",
+			},
+		}
+		zkPodAntiAffinity := apiv1.PodAntiAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: zkRequiredDuringSchedulingIgnoredDuringExecution,
+		}
+		zkAffinity := apiv1.Affinity{
+			PodAntiAffinity: &zkPodAntiAffinity,
+		}
+		zk.Spec.Template.Spec.Affinity = &zkAffinity
+	}
+
+	// we want to ensure the replica count is always at 3
 	replicaCount := int32(ZookeeperReplicas)
 
 	// If we are bumping from 1 to 3 replicas we need to backup zookeeper first.

controllers/zookeeper.go

@@ -8,11 +8,16 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - Deploy zookeeper as a 3-replica ensemble to ensure high availability. Ensure you have enough resources on the cluster to deploy 3 zookeeper pods.
-  While the process handles snapshotting Zookeeper's data and restoring it to all pods after the extra replicas rollout, you are **strongly** advised
-  to perform a backup with the sf-operator CLI before upgrading to avoid accidental data loss.
-  PKI certificates will be recreated to reflect the new ensemble setup.
+  While the upgrade process handles snapshotting Zookeeper's data and restoring it to all pods after the extra replicas rollout,
+  **You are strongly advised to backup your zookeeper data before upgrading to avoid accidental data loss.**
+  The PKI certificates will be recreated to reflect the new ensemble setup, and the zookeeper statefulset will be re-rolled out with its updated
+  configuration.
+  It is advised to shut down any zookeeper clients (specifically zuul executors) prior to the upgrade to avoid connection loss exceptions.
 - Added `PodDisruptionBudget` and `RollingUpdate` strategy to the Zookeeper controller, in accordance with zookeeper
-  being now deployed as a 3-replica ensemble (see below).
+  being now deployed as a 3-replica ensemble.
+- Added a PodAntiAffinity toggle for Zookeeper in the Custom Resource Definition (disabled by default). If enabled, the Zookeeper
+  statefulset will be configured so that its pods spawn on distinct worker nodes. Make sure your deployment cluster has at least three
+  available nodes, otherwise Software Factory will fail to reconcile when the toggle is enabled.
 
 ### Deprecated
 ### Removed

docs/reference/CHANGELOG.md

@@ -549,6 +549,7 @@ _Appears in:_
 | --- | --- | --- |
 | `storage` _[StorageSpec](#storagespec)_ |  | -|
 | `limits` _[LimitsSpec](#limitsspec)_ | Memory/CPU Limit | {map[cpu:500m memory:2Gi]}|
+| `nodeAntiAffinityEnabled` _boolean_ | Enable node anti-affinity, defaults to false so as not to fail on minikube deployments. This should be set to true on production deployments with at least 3 nodes available. | {false}|
 
 
 #### ZuulExecutorSpec