Fix redis split-brain after pod-0 restart during failover

When redis-redis-0 (the bootstrap pod) is deleted during a failover,
it restarts and tries to contact sentinel to find the current master.
Three problems caused it to fall through to the bootstrap path and
start a new independent master, creating a split-brain:

1. Single-try timeout: if sentinel was momentarily unreachable (e.g.
   the sentinel container on pod-0 itself was still starting), the
   3-second timeout expired and pod-0 immediately bootstrapped.

2. Headless service DNS: with PublishNotReadyAddresses: true, the
   headless service DNS can resolve to pod-0's own IP, so redis-cli
   connects to its own uninitialized sentinel instead of a peer.

3. Stale master identity: even when contacting a peer sentinel, it
   may still report the restarting pod as master (within the
   down-after-milliseconds window before failover completes).

Fix by adding a wait_for_master() function in common.sh that:
- Contacts each peer pod individually by FQDN (skipping self)
- Uses the REPLICAS env var to iterate only over actual peers
- Retries up to 10 times (30s total) before allowing bootstrap
- Rejects answers where the peer still thinks we are master
- Returns immediately if no peers are reachable at all (e.g. first
  deployment), avoiding unnecessary delay on initial bootstrap

Also pass a REPLICAS env var from the StatefulSet spec so the
script knows the exact replica count, increase InitialDelaySeconds
to 40s on all redis and sentinel probes so Kubernetes doesn't kill
the pod before the retry loop completes, and remove unused TCP
probe variables that were never referenced by the redis container.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: lmiccini

internal/redis/statefulset.go

@@ -27,36 +27,15 @@ func StatefulSet(
 	}
 	ls := labels.GetLabels(r, "redis", matchls)
 
-	livenessProbe := &corev1.Probe{
-		// TODO might need tuning
-		TimeoutSeconds:      5,
-		PeriodSeconds:       3,
-		InitialDelaySeconds: 3,
-	}
-	readinessProbe := &corev1.Probe{
-		// TODO might need tuning
-		TimeoutSeconds:      5,
-		PeriodSeconds:       5,
-		InitialDelaySeconds: 5,
-	}
 	sentinelLivenessProbe := &corev1.Probe{
-		// TODO might need tuning
 		TimeoutSeconds:      5,
 		PeriodSeconds:       3,
-		InitialDelaySeconds: 3,
+		InitialDelaySeconds: 40,
 	}
 	sentinelReadinessProbe := &corev1.Probe{
-		// TODO might need tuning
 		TimeoutSeconds:      5,
 		PeriodSeconds:       5,
-		InitialDelaySeconds: 5,
-	}
-
-	livenessProbe.TCPSocket = &corev1.TCPSocketAction{
-		Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(6379)},
-	}
-	readinessProbe.TCPSocket = &corev1.TCPSocketAction{
-		Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(6379)},
+		InitialDelaySeconds: 40,
 	}
 	sentinelLivenessProbe.TCPSocket = &corev1.TCPSocketAction{
 		Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(26379)},
@@ -78,6 +57,9 @@ func StatefulSet(
 	}, {
 		Name:  "CONFIG_HASH",
 		Value: configHash,
+	}, {
+		Name:  "REPLICAS",
+		Value: strconv.Itoa(int(*r.Spec.Replicas)),
 	}}
 
 	sts := &appsv1.StatefulSet{
@@ -115,13 +97,15 @@ func StatefulSet(
 										Command: []string{"/var/lib/operator-scripts/redis_probe.sh", "liveness"},
 									},
 								},
+								InitialDelaySeconds: 40,
 							},
 							ReadinessProbe: &corev1.Probe{
 								ProbeHandler: corev1.ProbeHandler{
 									Exec: &corev1.ExecAction{
 										Command: []string{"/var/lib/operator-scripts/redis_probe.sh", "readiness"},
 									},
 								},
+								InitialDelaySeconds: 40,
 							},
 						}, {
 							Image:   r.Spec.ContainerImage,

templates/redis/bin/common.sh

@@ -84,6 +84,52 @@ function remove_pod_label() {
     configure_pod_label $pod "$patch" "(200|422)"
 }
 
+# Wait for a peer sentinel to report a valid master for the cluster.
+# Contacts each peer pod individually by FQDN (skipping self) to avoid
+# the headless service DNS resolving to our own uninitialized sentinel.
+# If a peer still reports US as master (stale info before
+# down-after-milliseconds triggers failover), keeps retrying until
+# failover completes and a different master is elected.
+# If no peers are reachable at all (first deployment), returns
+# immediately so the bootstrap pod can start without delay.
+# Prints the master address on success (FQDN or IP).
+function wait_for_master() {
+    local retries=${SENTINEL_RETRIES:-10}
+    local delay=${SENTINEL_RETRY_DELAY:-3}
+    local pod_ordinal=${POD_NAME##*-}
+    local pod_base=${POD_NAME%-*}
+    local replicas=${REPLICAS:-3}
+
+    for i in $(seq 1 $retries); do
+        local any_peer_reachable=0
+        local ordinal=0
+        while [ $ordinal -lt $replicas ]; do
+            if [ "$ordinal" != "$pod_ordinal" ]; then
+                local peer="${pod_base}-${ordinal}.${SVC_FQDN}"
+                local output
+                if output=$(timeout ${TIMEOUT} $REDIS_CLI_CMD -h ${peer} -p 26379 sentinel master redis 2>/dev/null); then
+                    any_peer_reachable=1
+                    local master
+                    master=$(echo "$output" | awk '/^ip$/ {getline; print $0; exit}')
+                    # If the peer still thinks WE are master, it has stale
+                    # pre-failover info — try remaining peers before waiting.
+                    if ! echo "$master" | grep -q "^${POD_NAME}\."; then
+                        echo "$master"
+                        return 0
+                    fi
+                fi
+            fi
+            ordinal=$((ordinal + 1))
+        done
+        if [ $any_peer_reachable -eq 0 ]; then
+            return 1
+        fi
+        log "Attempt $i/$retries: no valid master found, retrying in ${delay}s..."
+        sleep $delay
+    done
+    return 1
+}
+
 function set_pod_label() {
     local pod="$1"
     local label="$2"

templates/redis/bin/start_redis_replication.sh

@@ -5,11 +5,9 @@
 generate_configs
 sudo -E kolla_set_configs
 
-# 1. check if a redis cluster is already running by contacting sentinel
-output=$(timeout ${TIMEOUT} $REDIS_CLI_CMD -h ${SVC_FQDN} -p 26379 sentinel master redis)
+# 1. check if a redis cluster is already running by contacting peer sentinels
+master=$(wait_for_master)
 if [ $? -eq 0 ]; then
-    master=$(echo "$output" | awk '/^ip$/ {getline; print $0; exit}')
-    # TODO skip if no master was found
     log "Connecting to the existing Redis cluster (master: ${master})"
     exec redis-server $REDIS_CONFIG --protected-mode no --replicaof "$master" 6379
 fi

templates/redis/bin/start_sentinel.sh

@@ -5,11 +5,9 @@
 generate_configs
 sudo -E kolla_set_configs
 
-# 1. check if a redis cluster is already running by contacting sentinel
-output=$(timeout ${TIMEOUT} $REDIS_CLI_CMD -h ${SVC_FQDN} -p 26379 sentinel master redis)
+# 1. check if a redis cluster is already running by contacting peer sentinels
+master=$(wait_for_master)
 if [ $? -eq 0 ]; then
-    master=$(echo "$output" | awk '/^ip$/ {getline; print $0; exit}')
-    # TODO skip if no master was found
     log "Connecting to the existing sentinel cluster (master: $master)"
     echo "sentinel monitor redis ${master} 6379 ${SENTINEL_QUORUM}" >> $SENTINEL_CONFIG
     exec redis-sentinel $SENTINEL_CONFIG