wsrep_notify: only probe the root credentials when the server is initialized

When a mysql server starts with galera enabled, the galera library
performs actions before the InnoDB engine is fully initialized and
the server is ready to process SQL commands. Until then, no
script should probe the mysql server.

Ensure that mysql_wsrep_notify does not probe root credentials before
it knows the server mysql server is fully initialized, i.e. when
the script knows that the local node is synced (primary partition).

In addition, to prevent other scripts from getting stalled while
probling the mysql server (e.g. when a remote node cannot be reached),
add a timeout period in the credential check logic.

Jira: OSPRH-14916

Author: dciabrin

templates/galera/bin/mysql_root_auth.sh

@@ -17,6 +17,8 @@ GALERA_INSTANCE="{{.galeraInstanceName}}"
 MY_CNF="$HOME/.my.cnf"
 MYSQL_SOCKET=/var/lib/mysql/mysql.sock
 
+CREDENTIALS_CHECK_TIMEOUT=4
+
 # Set up connection parameters based on whether we're connecting remotely or locally
 if [ -n "${MYSQL_REMOTE_HOST}" ]; then
 
@@ -45,22 +47,27 @@ if [ -f "${MY_CNF}" ]; then
             SHOULD_VALIDATE=true
         fi
 
-        if [ "${SHOULD_VALIDATE}" = "true" ] && mysql ${MYSQL_CONN_PARAMS} -uroot -p"${PASSWORD}" -e "SELECT 1;" >/dev/null 2>&1; then
-            # Credentials are still valid, use cached values
+        credentials_check=1
+        if [ "${SHOULD_VALIDATE}" = "true" ]; then
+            timeout ${CREDENTIALS_CHECK_TIMEOUT} mysql ${MYSQL_CONN_PARAMS} -uroot -p"${PASSWORD}" -e "SELECT 1;" >/dev/null 2>&1
+            credentials_check=$?
+        fi
 
-            MYSQL_PWD="${PASSWORD}"
-            DB_ROOT_PASSWORD="${PASSWORD}"
-            export MYSQL_PWD
-            export DB_ROOT_PASSWORD
-            return 0 2>/dev/null || exit 0
+        if [ "${SHOULD_VALIDATE}" = "true" ] && [ $credentials_check -eq 124 ]; then
+            # MySQL validation timed out, assume cache is valid and will be validated on next probe
+            export MYSQL_PWD="${PASSWORD}"
+            export DB_ROOT_PASSWORD="${PASSWORD}"
+            return 0
+        elif [ "${SHOULD_VALIDATE}" = "true" ] && [ $credentials_check -eq 0 ]; then
+            # Credentials are still valid, use cached values
+            export MYSQL_PWD="${PASSWORD}"
+            export DB_ROOT_PASSWORD="${PASSWORD}"
+            return 0
         elif [ "${USE_SOCKET}" = "true" ] && [ ! -S "${MYSQL_SOCKET}" ]; then
             # MySQL not running locally, assume cache is valid and will be validated on next probe
-
-            MYSQL_PWD="${PASSWORD}"
-            DB_ROOT_PASSWORD="${PASSWORD}"
-            export MYSQL_PWD
-            export DB_ROOT_PASSWORD
-            return 0 2>/dev/null || exit 0
+            export MYSQL_PWD="${PASSWORD}"
+            export DB_ROOT_PASSWORD="${PASSWORD}"
+            return 0
         fi
     fi
     # If we get here, credentials are invalid, fall through to refresh

templates/galera/bin/mysql_wsrep_notify.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-source /var/lib/operator-scripts/mysql_root_auth.sh
-
 # NOTE(dciabrin) we might use downward API to populate those in the future
 PODNAME=$HOSTNAME
 SERVICE=${PODNAME/-galera-[0-9]*/}
@@ -293,6 +291,9 @@ fi
 
 # Contition: ask for a failover. This should be called when mysql is running
 if echo "${STATUS}" | grep -i -q -e 'failover'; then
+    # note: make sure that the root credentials are up to date
+    # before invoking any mysql command
+    source /var/lib/operator-scripts/mysql_root_auth.sh
     mysql_probe_state
     if [ $? != 0 ]; then
         log_error "Could not probe missing mysql information. Aborting"
@@ -312,6 +313,10 @@ if echo "${STATUS}" | grep -i -q -v -e 'synced'; then
 fi
 
 # At this point mysql is started, query missing arguments
+
+# note: make sure that the root credentials are up to date
+# before invoking any mysql command
+source /var/lib/operator-scripts/mysql_root_auth.sh
 mysql_probe_state
 if [ $? != 0 ]; then
     log_error "Could not probe missing mysql information. Aborting"