fix(redis-ha): split-brain script race condition and silent quorum failure (#404)

* fix(redis-ha): fix split-brain script race condition and silent quorum failure

Fixes #383 and #398 in the fix-split-brain.sh template.

https://claude.ai/code/session_015PLT3F82jivLQrxjfV9Hgg

* Chart Update

Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com>

---------

Signed-off-by: Aaron Layfield <aaron.layfield@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>

Author: DandyDeveloper

charts/redis-ha/Chart.yaml

@@ -5,7 +5,7 @@ keywords:
 - redis
 - keyvalue
 - database
-version: 4.36.2
+version: 4.37.0
 appVersion: 8.2.4
 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management
 icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png

charts/redis-ha/templates/_configs.tpl

@@ -501,6 +501,9 @@
         identify_announce_ip
     done
 
+    QUORUM_FAIL_COUNT=0
+    MAX_QUORUM_FAILURES=${MAX_QUORUM_FAILURES:-5}
+
     trap "exit 0" TERM
     while true; do
         sleep {{ .Values.splitBrainDetection.interval }}
@@ -509,6 +512,7 @@
         identify_master
 
         if [ "$MASTER" = "$ANNOUNCE_IP" ]; then
+            QUORUM_FAIL_COUNT=0
             redis_role
             if [ "$ROLE" != "master" ]; then
                 echo "waiting for redis to become master"
@@ -522,17 +526,34 @@
                 fi
             fi
         elif [ "${MASTER}" ]; then
+            QUORUM_FAIL_COUNT=0
             identify_redis_master
             if [ "$REDIS_MASTER" != "$MASTER" ]; then
                 echo "Redis master and local master are not the same. waiting."
                 sleep {{ .Values.splitBrainDetection.retryInterval }}
                 identify_master
-                identify_redis_master
-                echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}. No need to reinitialize."
-                if [ "${REDIS_MASTER}" != "${MASTER}" ]; then
-                    echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}, reinitializing"
-                    reinit
+                if [ "$MASTER" = "$ANNOUNCE_IP" ]; then
+                    echo "This pod became master during wait, skipping reinit"
+                else
+                    identify_redis_master
+                    echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}. No need to reinitialize."
+                    if [ "${REDIS_MASTER}" != "${MASTER}" ]; then
+                        echo "Redis master is ${MASTER}, expected master is ${REDIS_MASTER}, reinitializing"
+                        reinit
+                    fi
+                fi
+            fi
+        else
+            QUORUM_FAIL_COUNT=$((QUORUM_FAIL_COUNT + 1))
+            echo "WARNING: Sentinel returned no master (quorum may be broken). Failure count: $QUORUM_FAIL_COUNT/$MAX_QUORUM_FAILURES"
+            if [ "$QUORUM_FAIL_COUNT" -ge "$MAX_QUORUM_FAILURES" ]; then
+                echo "ERROR: Quorum broken for $MAX_QUORUM_FAILURES consecutive checks. Attempting sentinel reset..."
+                if [ "$SENTINEL_PORT" -eq 0 ]; then
+                    redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel reset "${MASTER_GROUP}" || true
+                else
+                    redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel reset "${MASTER_GROUP}" || true
                 fi
+                QUORUM_FAIL_COUNT=0
             fi
         fi
     done