K8SPS-69 | async cluster readiness check should account for running backup (#1292)

* check backup status in async readiness check

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* check that backups are enabled first

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* fix nil pointer

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* fix e2e tests

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* use env constant

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* update unit tests

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

* check backup running only when replication is stopped

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

---------

Signed-off-by: Mayank Shah <mayank.shah@percona.com>

Author: mayankshah1607

cmd/healthcheck/main.go

@@ -18,6 +18,7 @@ import (
 	mysqldb "github.com/percona/percona-server-mysql-operator/pkg/db"
 	"github.com/percona/percona-server-mysql-operator/pkg/k8s"
 	"github.com/percona/percona-server-mysql-operator/pkg/naming"
+	"github.com/percona/percona-server-mysql-operator/pkg/xtrabackup"
 )
 
 const (
@@ -130,7 +131,12 @@ func checkReadinessAsync(ctx context.Context) error {
 	case replStatus == mysqldb.ReplicationStatusActive && !readOnly:
 		return errors.New("replica is not read only")
 	case replStatus == mysqldb.ReplicationStatusStopped:
-		return errors.New("replication is stopped")
+		// If replication is stopped, check if it is because of a running backup
+		if running, err := isBackupRunning(ctx); err != nil {
+			return errors.Wrap(err, "check backup running")
+		} else if !running {
+			return errors.New("replication is stopped")
+		}
 	}
 	return nil
 }
@@ -327,3 +333,16 @@ func fileExists(name string) (bool, error) {
 	}
 	return true, nil
 }
+
+func isBackupRunning(ctx context.Context) (bool, error) {
+	backupsEnabled := os.Getenv(naming.EnvBackupsEnabled)
+	if backupsEnabled != "true" {
+		return false, nil
+	}
+	sc := xtrabackup.NewSidecarClient("localhost")
+	bcp, err := sc.GetRunningBackupConfig(ctx)
+	if err != nil {
+		return false, errors.Wrap(err, "get running backup config")
+	}
+	return bcp != nil, nil
+}

e2e-tests/tests/limits/01-assert.yaml

@@ -74,6 +74,8 @@ spec:
           value: /var/lib/mysql/mysql.state
         - name: KEYRING_VAULT_PATH
           value: /etc/mysql/vault-keyring-secret/keyring_vault.cnf
+        - name: BACKUPS_ENABLED
+          value: "true"
         imagePullPolicy: Always
         livenessProbe:
           exec:

e2e-tests/tests/limits/03-assert.yaml

@@ -74,6 +74,8 @@ spec:
           value: /var/lib/mysql/mysql.state
         - name: KEYRING_VAULT_PATH
           value: /etc/mysql/vault-keyring-secret/keyring_vault.cnf
+        - name: BACKUPS_ENABLED
+          value: "true"
         imagePullPolicy: Always
         livenessProbe:
           exec:

e2e-tests/tests/limits/05-assert.yaml

@@ -74,6 +74,8 @@ spec:
           value: /var/lib/mysql/mysql.state
         - name: KEYRING_VAULT_PATH
           value: /etc/mysql/vault-keyring-secret/keyring_vault.cnf
+        - name: BACKUPS_ENABLED
+          value: "true"
         imagePullPolicy: Always
         livenessProbe:
           exec:

pkg/mysql/mysql.go

@@ -679,6 +679,17 @@ func mysqldContainer(cr *apiv1.PerconaServerMySQL) corev1.Container {
 		})
 	}
 
+	if cr.CompareVersion("1.1.0") >= 0 {
+		backupsEnabled := false
+		if cr.Spec.Backup != nil {
+			backupsEnabled = cr.Spec.Backup.Enabled
+		}
+		env = append(env, corev1.EnvVar{
+			Name:  naming.EnvBackupsEnabled,
+			Value: strconv.FormatBool(backupsEnabled),
+		})
+	}
+
 	container := corev1.Container{
 		Name:                     AppName,
 		Image:                    spec.Image,

pkg/mysql/mysql_test.go

@@ -160,6 +160,29 @@ func TestStatefulSet(t *testing.T) {
 		assert.Equal(t, "mysql-annotation", sts.Spec.Template.Annotations["mysql-annotation"])
 
 	})
+
+	t.Run("env variables", func(t *testing.T) {
+		cluster := cr.DeepCopy()
+		cluster.Spec.Backup = &apiv1.BackupSpec{
+			Enabled: true,
+		}
+		sts := StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+
+		envs := sts.Spec.Template.Spec.Containers[0].Env
+		assert.Contains(t, envs, corev1.EnvVar{
+			Name:  naming.EnvBackupsEnabled,
+			Value: "true",
+		})
+
+		// backups disabled
+		cluster.Spec.Backup.Enabled = false
+		sts = StatefulSet(cluster, initImage, configHash, tlsHash, secret)
+		envs = sts.Spec.Template.Spec.Containers[0].Env
+		assert.Contains(t, envs, corev1.EnvVar{
+			Name:  naming.EnvBackupsEnabled,
+			Value: "false",
+		})
+	})
 }
 
 func TestStatefulsetVolumes(t *testing.T) {

pkg/naming/env.go

@@ -15,4 +15,6 @@ const (
 	EnvAsyncSourceRetryCount = "ASYNC_SOURCE_RETRY_COUNT"
 
 	EnvAsyncSourceConnectRetry = "ASYNC_SOURCE_CONNECT_RETRY"
+
+	EnvBackupsEnabled = "BACKUPS_ENABLED"
 )