integrationt test for sticky vol failover failback

Rohit-PX · Rohit-PX · commit b50f2f8dc630 · 2023-05-05T01:38:31.000Z
Signed-off-by: Rohit-PX &lt;rkulkarni@purestorage.com&gt;
diff --git a/test/integration_test/migration_failover_failback_test.go b/test/integration_test/migration_failover_failback_test.go
@@ -6,11 +6,14 @@ package integrationtest
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
 	"github.com/portworx/sched-ops/k8s/core"
+	storkops "github.com/portworx/sched-ops/k8s/stork"
 	"github.com/portworx/sched-ops/task"
 	"github.com/portworx/torpedo/drivers/scheduler"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -54,10 +57,11 @@ func testMigrationFailoverFailback(t *testing.T) {
 
 	t.Run("vanillaFailoverAndFailbackMigrationTest", vanillaFailoverAndFailbackMigrationTest)
 	t.Run("rancherFailoverAndFailbackMigrationTest", rancherFailoverAndFailbackMigrationTest)
+	t.Run("stickyFlagFailoverAndFailbackMigrationTest", stickyFlagFailoverAndFailbackMigrationTest)
 }
 
 func vanillaFailoverAndFailbackMigrationTest(t *testing.T) {
-	failoverAndFailbackMigrationTest(t)
+	failoverAndFailbackMigrationTest(t, "mysql-enc-pvc", "mysql-migration-failover-failback", true)
 }
 
 func rancherFailoverAndFailbackMigrationTest(t *testing.T) {
@@ -105,20 +109,22 @@ func rancherFailoverAndFailbackMigrationTest(t *testing.T) {
 
 	scaleFactor := testMigrationFailover(t, preMigrationCtx, ctxs, "", appKey, instanceID)
 
-	testMigrationFailback(t, preMigrationCtx, ctxs, scaleFactor, projectIDMappingsReverse, appKey, instanceID)
+	testMigrationFailback(t, preMigrationCtx, ctxs, scaleFactor, projectIDMappingsReverse, appKey, instanceID, true)
 }
 
-func failoverAndFailbackMigrationTest(t *testing.T) {
+func stickyFlagFailoverAndFailbackMigrationTest(t *testing.T) {
+	failoverAndFailbackMigrationTest(t, "mysql-sticky", "mysql-migration-sticky", false)
+}
+
+func failoverAndFailbackMigrationTest(t *testing.T, appKey, migrationKey string, failbackSuccessExpected bool) {
 
-	appKey := "mysql-enc-pvc"
-	instanceID := "mysql-migration-failover-failback"
 	// Migrate the resources
 	ctxs, preMigrationCtx := triggerMigration(
 		t,
-		instanceID,
+		migrationKey,
 		appKey,
 		nil,
-		[]string{instanceID},
+		[]string{migrationKey},
 		true,
 		false,
 		false,
@@ -150,9 +156,9 @@ func failoverAndFailbackMigrationTest(t *testing.T) {
 	// validate the migration summary based on the application specs that were deployed by the test
 	validateMigrationSummary(t, preMigrationCtx, expectedResources, expectedVolumes, migrationObj.Name, migrationObj.Namespace)
 
-	scaleFactor := testMigrationFailover(t, preMigrationCtx, ctxs, "", appKey, instanceID)
+	scaleFactor := testMigrationFailover(t, preMigrationCtx, ctxs, "", appKey, migrationKey)
 
-	testMigrationFailback(t, preMigrationCtx, ctxs, scaleFactor, "", appKey, instanceID)
+	testMigrationFailback(t, preMigrationCtx, ctxs, scaleFactor, "", appKey, migrationKey, failbackSuccessExpected)
 }
 
 func testMigrationFailover(
@@ -242,6 +248,7 @@ func testMigrationFailback(
 	scaleFactor map[string]int32,
 	projectIDMappings string,
 	appKey, instanceID string,
+	failbackSuccessExpected bool,
 ) {
 	// Failback the application
 	// Trigger a reverse migration
@@ -251,6 +258,8 @@ func testMigrationFailback(
 	require.NoError(t, err, "Error scheduling task")
 	require.Equal(t, 1, len(ctxsReverse), "Only one task should have started")
 
+	appCtx := ctxsReverse[0]
+
 	err = schedulerDriver.WaitForRunning(ctxsReverse[0], defaultWaitTimeout, defaultWaitInterval)
 	require.NoError(t, err, "Error waiting for app to get to running state")
 
@@ -265,6 +274,50 @@ func testMigrationFailback(
 		scheduler.ScheduleOptions{AppKeys: []string{instanceID}})
 	require.NoError(t, err, "Error scheduling migration specs")
 
+	if !failbackSuccessExpected {
+		// In case of sticky volumes migration failure is expected, so will update volume and trigger migration again
+		err = schedulerDriver.WaitForRunning(ctxsReverse[0], defaultWaitTimeout/10, defaultWaitInterval)
+		require.Error(t, err, "Expected failback migration to fail")
+
+		// Get volumes for this migration on source cluster and update sticky flag
+		err = setSourceKubeConfig()
+		require.NoError(t, err, "Error resetting source config, for updating sticky volume")
+
+		vols, err := schedulerDriver.GetVolumes(appCtx)
+		require.NoError(t, err, "Error getting volumes for app")
+		for _, v := range vols {
+			err = volumeDriver.UpdateStickyFlag(v.ID, "off")
+			require.NoError(t, err, "Error updating sticky flag for volumes %s", v.Name)
+		}
+		time.Sleep(3 * time.Minute)
+
+		// Trigger migration on destination cluster again
+		err = setDestinationKubeConfig()
+		require.NoError(t, err, "Error setting destination config after updating sticky volume")
+
+		var failedMigrationObj *v1alpha1.Migration
+		var ok bool
+		for _, specObj := range ctxsReverse[0].App.SpecList {
+			if failedMigrationObj, ok = specObj.(*v1alpha1.Migration); ok {
+				break
+			}
+		}
+		failedMigrationObj, err = storkops.Instance().GetMigration(failedMigrationObj.Name, failedMigrationObj.Namespace)
+		require.NoError(t, err, "Error getting the failed migration")
+		logrus.Infof("Failed migration object found: %s in namespace: %s. Status: %s", failedMigrationObj.Name, failedMigrationObj.Namespace, failedMigrationObj.Status.Status)
+
+		err = deleteMigrations([]*v1alpha1.Migration{failedMigrationObj})
+		require.NoError(t, err, "error in deleting failed migrations.")
+
+		// apply migration specs again, it should pass this time
+		err = schedulerDriver.AddTasks(ctxsReverse[0],
+			scheduler.ScheduleOptions{AppKeys: []string{instanceID}})
+		require.NoError(t, err, "Error scheduling migration specs")
+
+		err = schedulerDriver.WaitForRunning(ctxsReverse[0], defaultWaitTimeout, defaultWaitInterval)
+		require.NoError(t, err, "Error waiting for migration to complete post sticky flag update")
+	}
+
 	err = schedulerDriver.WaitForRunning(ctxsReverse[0], defaultWaitTimeout, defaultWaitInterval)
 	require.NoError(t, err, "Error waiting for migration to complete")
 
@@ -322,8 +375,16 @@ func testMigrationFailback(
 			require.Equal(t, projectValue, "project-A")
 		}
 	}
-
 	destroyAndWait(t, []*scheduler.Context{postMigrationCtx})
+	destroyAndWait(t, ctxs)
+
+	err = setDestinationKubeConfig()
+	require.NoError(t, err, "Error resetting remote config")
+	destroyAndWait(t, ctxsReverse)
+
+	err = setSourceKubeConfig()
+	require.NoError(t, err, "Error resetting remote config")
+
 }
 
 // The below two functions are currently not invoked during the tests since the namespaceSelector
diff --git a/test/integration_test/specs/mysql-migration-sticky/migration.yaml b/test/integration_test/specs/mysql-migration-sticky/migration.yaml
@@ -0,0 +1,14 @@
+apiVersion: stork.libopenstorage.org/v1alpha1
+kind: Migration
+metadata:
+  name: mysql-sticky-migration
+spec:
+  # This should be the name of the cluster pair
+  clusterPair: remoteclusterpair
+  # If set to false this will migrate only the volumes. No PVCs, apps, etc will be migrated
+  includeResources: true
+  # If set to false, the deployments and stateful set replicas will be set to 0 on the destination. There will be an annotation with "stork.openstorage.org/migrationReplicas" to store the replica count from the source
+  startApplications: false
+  # List of namespaces to migrate
+  namespaces:
+  - mysql-sticky-mysql-migration-sticky 
diff --git a/test/integration_test/specs/mysql-sticky/aws/aws-sc.yaml b/test/integration_test/specs/mysql-sticky/aws/aws-sc.yaml
@@ -0,0 +1,8 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: mysql-sc 
+provisioner: kubernetes.io/aws-ebs
+parameters:
+  type: gp2
+reclaimPolicy: Delete
diff --git a/test/integration_test/specs/mysql-sticky/azure/azure-sc.yaml b/test/integration_test/specs/mysql-sticky/azure/azure-sc.yaml
@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: mysql-sc
+provisioner: kubernetes.io/azure-disk
+parameters:
+  skuName: Standard_LRS
diff --git a/test/integration_test/specs/mysql-sticky/gce/gke-sc.yaml b/test/integration_test/specs/mysql-sticky/gce/gke-sc.yaml
@@ -0,0 +1,10 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: mysql-sc
+parameters:
+  type: pd-standard
+provisioner: kubernetes.io/gce-pd
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
diff --git a/test/integration_test/specs/mysql-sticky/linstor/linstor-sc.yaml b/test/integration_test/specs/mysql-sticky/linstor/linstor-sc.yaml
@@ -0,0 +1,10 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: mysql-sc
+provisioner: linstor.csi.linbit.com
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+parameters:
+  autoPlace: "2"
+  storagePool: sda
diff --git a/test/integration_test/specs/mysql-sticky/mysql.yaml b/test/integration_test/specs/mysql-sticky/mysql.yaml
@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: mysql
+  labels:
+    app: mysql
+spec:
+  serviceName: mysql-service
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mysql
+      version: "1"
+  template:
+    metadata:
+      labels:
+        app: mysql
+        version: "1"
+    spec:
+      schedulerName: stork
+      containers:
+      - image: mysql:5.6
+        name: mysql
+        env:
+        - name: MYSQL_ROOT_PASSWORD
+          value: password
+        ports:
+        - containerPort: 3306
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "mysqladmin -u root -p$MYSQL_ROOT_PASSWORD ping"]
+          initialDelaySeconds: 70
+          periodSeconds: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "mysql -u root -p$MYSQL_ROOT_PASSWORD -e \"select 1\""]
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 5
+        volumeMounts:
+        - name: mysql-data
+          mountPath: /var/lib/mysql
+  volumeClaimTemplates:
+  - metadata:
+      name: mysql-data
+      annotations:
+        volume.beta.kubernetes.io/storage-class: mysql-sticky-sc
+        px/secret-name: volume-secrets
+        px/secret-namespace: kube-system
+        px/secret-key: mysql-secret
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 8Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql-service
+  labels:
+    app: mysql
+spec:
+  selector:
+    app: mysql
+  ports:
+    - name: transport
+      port: 3306
diff --git a/test/integration_test/specs/mysql-sticky/portworx/sc-sticky.yaml b/test/integration_test/specs/mysql-sticky/portworx/sc-sticky.yaml
@@ -0,0 +1,8 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+    name: mysql-sticky-sc
+provisioner: kubernetes.io/portworx-volume
+parameters:
+   repl: "2"
+   sticky: "true"
