e2e: add nvmeof GroupLock stress tests for concurrent operations

Add e2e tests to validate nvmeof NodeServer GroupLock implementation
under concurrent NodeStage (Group A) and NodeUnstage (Group B)
operations. The tests ensure no deadlock occurs when multiple PVCs and
Pods are created and deleted simultaneously.

New helper file (nvmeof_helper.go) provides reusable functions for
concurrent PVC/Pod operations with proper error tracking.
Two test cases cover:
1) sequential concurrent batches (create all, then delete
all)

2) mixed operations with pre-created batch to guarantee
continuous Group A/B switching..

Signed-off-by: gadi-didi <gadi.didi@ibm.com>

Author: gadididi

e2e/nvmeof.go

@@ -18,6 +18,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -189,5 +190,84 @@ var _ = ginkgo.Describe("nvmeof", func() {
 			validateRBDImageCount(f, 0, nvmeofPool)
 			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
 		})
+
+		ginkgo.It("Test GroupLock: Concurrent Create/Delete PVCs and Pods", func() {
+			// This test validates the GroupLock implementation in the NVMeoF NodeServer
+			// by creating and deleting multiple PVCs and Pods concurrently.
+			//
+			// Test flow:
+			// 1. Create 3 PVCs + Pods concurrently (triggers NodeStage -> Group A lock)
+			// 2. Wait for all to be ready
+			// 3. Delete all PVCs + Pods concurrently (triggers NodeUnstage -> Group B lock)
+			// 4. Verify no timeouts/deadlocks
+			// 5. Verify all operations succeed
+			totalCount := 3
+
+			ginkgo.By("Creating multiple PVCs and Pods concurrently")
+			createResult := createConcurrentPVCsAndApps(totalCount, pvcPath, appPath, f)
+
+			// Log any errors
+			if createResult.HasErrors() {
+				createResult.LogErrors()
+			}
+
+			// Verify all creations succeeded
+			Expect(createResult.failed).To(Equal(0),
+				"Expected all %d create operations to succeed, but %d failed",
+				totalCount, createResult.failed)
+
+			ginkgo.By("Validating backend RBD images were created")
+			validateRBDImageCount(f, totalCount, nvmeofPool)
+			validateOmapCount(f, totalCount, rbdType, nvmeofPool, volumesType)
+
+			ginkgo.By("Deleting multiple PVCs and Pods concurrently")
+			deleteResult := deleteConcurrentPVCsAndApps(createResult, pvcPath, appPath, f)
+
+			// Log any errors
+			if deleteResult.HasErrors() {
+				deleteResult.LogErrors()
+			}
+
+			// Verify all deletions succeeded
+			Expect(deleteResult.failed).To(Equal(0),
+				"Expected all %d delete operations to succeed, but %d failed",
+				totalCount, deleteResult.failed)
+
+			ginkgo.By("Validating all backend RBD images were deleted")
+			validateRBDImageCount(f, 0, nvmeofPool)
+			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
+
+			framework.Logf("GroupLock test passed: %d concurrent create and %d concurrent delete operations completed successfully",
+				totalCount, totalCount)
+		})
+
+		ginkgo.It("Test GroupLock: Mixed Create/Delete with Rapid Switching", func() {
+			// This test validates the GroupLock implementation under rapid switching
+			// between Group A (NodeStage) and Group B (NodeUnstage) operations.
+			//
+			// Test flow:
+			// 1. Create 5 PVCs + Pods (Group A)
+			// 2. Concurrently start creating next 5 PVCs + Pods (Group A) while deleting previous 5 (Group B)
+			// 3. Repeat for total of 15 PVCs + Pods with rapid switching between Group A and B
+			//
+			// This creates 15 total PVCs and forces rapid GroupLock switching,
+			// simulating "real-world" scenarios for catching potential deadlocks/starvation.
+			totalCount := 15
+			batchSize := 5
+
+			ginkgo.By(fmt.Sprintf("Running mixed create/delete test: %d total PVCs in batches of %d",
+				totalCount, batchSize))
+
+			err := mixedCreateDeletePVCsAndApps(totalCount, batchSize, pvcPath, appPath, f)
+			Expect(err).ShouldNot(HaveOccurred(),
+				"Mixed create/delete operations should complete without errors")
+
+			ginkgo.By("Validating all backend RBD images were cleaned up")
+			validateRBDImageCount(f, 0, nvmeofPool)
+			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
+
+			framework.Logf("GroupLock mixed test passed: %d PVCs created and deleted with rapid Group A/B switching",
+				totalCount)
+		})
 	})
 })