Improve ensure statefulset to wait for readiness and bailout on crash

This change updates the ensureStatefulset helpers to fully wait for
service readiness based on the graceful period

Change-Id: I8cf8d07608203546eb712518c6f7d1d493166733

Author: TristanCacqueray

controllers/git_server.go

@@ -370,7 +370,7 @@ func (r *SFController) DeployGitServer() bool {
 	svcRW := base.MkServicePod(GitServerIdentRW, r.Ns, GitServerIdent+"-0", []int32{gsGitPortRW}, gsGitPortName, r.cr.Spec.ExtraLabels)
 	r.EnsureService(&svcRW)
 
-	ready := r.IsStatefulSetReady(current)
+	ready := r.waitStatefulset(current)
 	conds.UpdateConditions(&r.cr.Status.Conditions, GitServerIdent, ready)
 
 	return ready

controllers/hound-search.go

@@ -133,7 +133,7 @@ func (r *SFController) DeployHoundSearch() bool {
 		houndSearchIdent+"-data-"+houndSearchIdent+"-0",
 		r.cr.Spec.Codesearch.Storage)
 
-	isReady := r.IsStatefulSetReady(current) && pvcReadiness
+	isReady := pvcReadiness && r.waitStatefulset(current)
 	conds.UpdateConditions(&r.cr.Status.Conditions, houndSearchIdent, isReady)
 
 	return isReady

controllers/logserver.go

@@ -257,7 +257,7 @@ func (r *SFController) DeployLogserver() bool {
 
 	pvcReadiness := r.reconcileExpandPVC(logserverIdent+"-"+logserverIdent+"-0", r.cr.Spec.Logserver.Storage)
 
-	isReady := r.IsStatefulSetReady(current) && !stsUpdated && pvcReadiness
+	isReady := !stsUpdated && pvcReadiness && r.waitStatefulset(current)
 
 	// When sts is ready, we can consider logserver ready and we can update the keys
 	// Reconcile SSH keys dynamically without pod restart

controllers/mariadb.go

@@ -278,7 +278,7 @@ GRANT ALL ON *.* TO root@'%%' WITH GRANT OPTION;`,
 
 	var zuulDBSecret apiv1.Secret
 
-	stsReady := r.IsStatefulSetReady(current)
+	stsReady := r.waitStatefulset(current)
 
 	postReady := false
 	if stsReady {

controllers/zookeeper.go

@@ -260,7 +260,7 @@ func (r *SFController) DeployZookeeper() bool {
 		pvcReadiness = pvcReadiness && r.reconcileExpandPVC(pvcName, r.cr.Spec.Zookeeper.Storage)
 	}
 
-	isReady := r.IsStatefulSetReady(current) && pvcReadiness
+	isReady := pvcReadiness && r.waitStatefulset(current)
 	conds.UpdateConditions(&r.cr.Status.Conditions, ZookeeperIdent, isReady)
 
 	return isReady

controllers/zuul.go

@@ -621,7 +621,7 @@ func (r *SFController) EnsureZuulScheduler(cfg *ini.File) bool {
 		return false
 	}
 
-	ready := r.IsStatefulSetReady(current)
+	ready := r.waitStatefulset(current)
 	conds.UpdateConditions(&r.cr.Status.Conditions, "zuul-scheduler", ready)
 
 	if ready {
@@ -797,7 +797,7 @@ func (r *SFController) EnsureZuulMerger(cfg *ini.File) bool {
 
 	pvcReadiness := r.reconcileExpandPVCs("zuul-merger", r.cr.Spec.Zuul.Merger.Storage)
 
-	ready := r.IsStatefulSetReady(current) && pvcReadiness
+	ready := pvcReadiness && r.waitStatefulset(current)
 	conds.UpdateConditions(&r.cr.Status.Conditions, service, ready)
 
 	return ready

docs/reference/CHANGELOG.md

@@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - Zuul Executor: wait until the grace period is over for the statefulset to be ready. This is to ensure progressive rollout where the sf-operator can now safely proceed one executors at a time without interrupting the production.
+- Improved statefulset readiness deployment to wait for the grace period and bail out early in case of crash back loop.
 
 ### Deprecated
 ### Removed