e2e: add VolumeAttributesClass example and e2e test

Add a VolumeAttributesClass example for NVMe-oF volumes
demonstrating the mutable parameters supported by the driver:
- Host access control via allowHostNQNs parameter (for external hosts)
- QoS parameters

The example includes a working configuration
with host access control enabled,
and all optional parameters documented
as comments with explanations.

Also add an e2e test that validates VAC functionality by:
1. Creating a VAC with allowHostNQNs parameter
2. Creating a PVC that references the VAC
3. Verifying volume creation
4. check the host was added by run nvme connect
5. activate new VAC and recheck the new host
5. Validating cleanup

Signed-off-by: gadi-didi <gadi.didi@ibm.com>

Author: gadididi

e2e/nvmeof-deploy.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"strings"
 	"time"
 
@@ -29,6 +30,7 @@ import (
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -168,7 +170,13 @@ func createNVMeoFStorageClass(
 		}
 	}
 
-	sc.Parameters["subsystemNQN"] = "nqn.2025-08.io.ceph:" + f.UniqueName
+	// Only set subsystemNQN if not explicitly disabled via skipSubsystemNQN parameter
+	// For external clients using allowHostNQNs, subsystemNQN should not be set
+	if _, skipSubsystemNQN := parameters["skipSubsystemNQN"]; !skipSubsystemNQN {
+		sc.Parameters["subsystemNQN"] = "nqn.2025-08.io.ceph:" + f.UniqueName
+	}
+	// Remove the skipSubsystemNQN marker from parameters as it's not a real SC parameter
+	delete(sc.Parameters, "skipSubsystemNQN")
 
 	gwHost, gwIP := getNVMeofGateway(f.ClientSet)
 	framework.Logf("configuring StorageClass for gateway %q at %s", gwHost, gwIP)
@@ -242,3 +250,73 @@ func createNVMeoFCredentials(f *framework.Framework) {
 	err = createRBDSecret(f, nvmeofNodePluginSecretName, nvmeofKeyringNodePluginUsername, key)
 	Expect(err).ShouldNot(HaveOccurred())
 }
+
+func createNVMeOFVolumeAttributesClass(
+	c kubernetes.Interface,
+	name string,
+	params map[string]string,
+) error {
+	vac, err := getVolumeAttributesClass(nvmeofExamplePath + "volumeattributesclass.yaml")
+	if err != nil {
+		return fmt.Errorf("failed to get vac: %w", err)
+	}
+	if name != "" {
+		vac.Name = name
+	}
+
+	// overload any parameters that were passed
+	if params == nil {
+		// create an empty params, so that params["clusterID"] below
+		// does not panic
+		params = map[string]string{}
+	}
+	maps.Copy(vac.Parameters, params)
+
+	timeout := time.Duration(deployTimeout) * time.Minute
+
+	return wait.PollUntilContextTimeout(context.TODO(), poll, timeout, true, func(ctx context.Context) (bool, error) {
+		_, err = c.StorageV1().VolumeAttributesClasses().Create(ctx, &vac, metav1.CreateOptions{})
+		if err != nil {
+			if apierrs.IsAlreadyExists(err) {
+				return true, nil
+			}
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			framework.Logf("error creating VolumeAttributesClass %q: %v", vac.Name, err)
+
+			return false, fmt.Errorf("failed to create VolumeAttributesClass %q: %w", vac.Name, err)
+		}
+
+		return true, nil
+	})
+}
+
+func deleteNVMeOFVolumeAttributesClass(c kubernetes.Interface, name string) error {
+	vac, err := getVolumeAttributesClass(nvmeofExamplePath + "volumeattributesclass.yaml")
+	if err != nil {
+		return err
+	}
+	if name != "" {
+		vac.Name = name
+	}
+
+	timeout := time.Duration(deployTimeout) * time.Minute
+
+	return wait.PollUntilContextTimeout(context.TODO(), poll, timeout, true, func(ctx context.Context) (bool, error) {
+		err = c.StorageV1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})
+		if err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			framework.Logf("error deleting VolumeAttributesClass %q: %v", vac.Name, err)
+
+			return false, fmt.Errorf("failed to delete VolumeAttributesClass %q: %w", vac.Name, err)
+		}
+
+		return true, nil
+	})
+}

e2e/nvmeof.go

@@ -311,5 +311,126 @@ var _ = ginkgo.Describe("nvmeof", func() {
 			framework.Logf("GroupLock Pods-only test passed: %d Pods created and deleted with rapid Group A/B switching",
 				totalCount)
 		})
+
+		ginkgo.It("create a PVC with VAC containing allowHostNQNs, apply new VAC then delete the PVC", func() {
+
+			// Vac test requires Kubernetes 1.34+ for VolumeAttributesClass support
+			if !k8sVersionGreaterEquals(f.ClientSet, 1, 34) {
+				framework.Logf("skipping VolumeAttributesClass test, needs Kubernetes >= 1.34")
+
+				return
+			}
+
+			// This test validates VolumeAttributesClass (VAC) support for NVMe-oF volumes
+			// with host access control via the allowHostNQNs parameter.
+			//
+			// Test flow:
+			// 1. Create a StorageClass for external clients (without subsystemNQN) - deleted in the end.
+			// 2. Create 2 VACs with allowHostNQNs parameter containing lists of allowed host NQNs
+			// 3. Create a PVC that references the VAC via spec.volumeAttributesClassName
+			// 4. Verify the volume is created and the host access control is applied
+			// 5. Update the PVC to reference a different VAC and verify the update is applied
+			// 6. Delete the PVC and verify cleanup
+			//
+			// This tests the ControllerModifyVolume functionality in the NVMe-oF driver,
+			// ensuring that host access lists can be configured via VAC at volume creation time.
+			// For external clients using allowHostNQNs, the StorageClass should not have subsystemNQN.
+
+			// Step 1: Create a separate StorageClass for external clients (without subsystemNQN)
+			externalClientSC := "e2e-" + f.UniqueName + "-sc-external"
+			options := map[string]string{}
+			params := map[string]string{
+				"pool":             nvmeofPool,
+				"skipSubsystemNQN": "true", // Signal to skip subsystemNQN for external clients
+			}
+			policy := v1.PersistentVolumeReclaimDelete
+
+			ginkgo.By("Creating StorageClass for external clients (without subsystemNQN)")
+			createNVMeoFStorageClass(f, externalClientSC, options, params, policy)
+			defer deleteNVMeofStorageClass(f, externalClientSC)
+
+			// Step 2: Create 2 VACs with allowHostNQNs parameter
+			vacName1 := "e2e-" + f.UniqueName + "-vac1"
+			vacName2 := "e2e-" + f.UniqueName + "-vac2"
+			hostListParam := map[string]string{
+				"allowHostNQNs": `- nqn.2014-08.org.nvmexpress:test-host-1
+- nqn.2014-08.org.nvmexpress:test-host-2
+- nqn.2014-08.org.nvmexpress:test-host-3`,
+			}
+			hostListParam2 := map[string]string{
+				"allowHostNQNs": `- nqn.2014-08.org.nvmexpress:test-host-1
+- nqn.2014-08.org.nvmexpress:test-host-5`,
+			}
+			ginkgo.By("Creating VolumeAttributesClass with allowHostNQNs")
+			err := createNVMeOFVolumeAttributesClass(f.ClientSet, vacName1, hostListParam)
+			Expect(err).ShouldNot(HaveOccurred())
+			defer func() {
+				err = deleteNVMeOFVolumeAttributesClass(f.ClientSet, vacName1)
+				if err != nil {
+					logAndFail("failed to delete volumeattributesclass: %v", err)
+				}
+			}()
+
+			ginkgo.By("Creating second VolumeAttributesClass with different allowHostNQNs")
+			err = createNVMeOFVolumeAttributesClass(f.ClientSet, vacName2, hostListParam2)
+			Expect(err).ShouldNot(HaveOccurred())
+			defer func() {
+				err = deleteNVMeOFVolumeAttributesClass(f.ClientSet, vacName2)
+				if err != nil {
+					logAndFail("failed to delete volumeattributesclass: %v", err)
+				}
+			}()
+
+			// Step 3: Create PVC with reference to the VAC1 and external client StorageClass
+			pvc, err := loadPVC(pvcPath)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			pvc.Namespace = f.UniqueName
+			pvc.Spec.StorageClassName = &externalClientSC // Use external client SC without subsystemNQN
+			pvc.Spec.VolumeAttributesClassName = &vacName1
+
+			ginkgo.By("Creating PVC with VolumeAttributesClass reference for external clients")
+			err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			pv, err := getBoundPV(f.ClientSet, pvc)
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(pv.Spec.VolumeAttributesClassName).NotTo(BeNil())
+			Expect(*pv.Spec.VolumeAttributesClassName).To(Equal(vacName1))
+
+			validateRBDImageCount(f, 1, nvmeofPool)
+			validateOmapCount(f, 1, rbdType, nvmeofPool, volumesType)
+
+			// Step 4: Verify allowHostNQNs by testing connection with allowed and disallowed hosts
+			ginkgo.By("Verifying allowHostNQNs configuration via nvme connect-all")
+			_, gatewayIP := getNVMeofGateway(f.ClientSet)
+			err = verifyAllowHostNQNsViaConnect(f, gatewayIP, hostListParam["allowHostNQNs"])
+			Expect(err).ShouldNot(HaveOccurred())
+
+			// Step 5: Update PVC to reference VAC2 and verify the update is applied
+			ginkgo.By("Updating PVC to reference a different VolumeAttributesClass")
+
+			err = modifyPVCVolumeAttributesClass(
+				f.ClientSet,
+				pvc,
+				vacName2)
+			if err != nil {
+				logAndFail("failed to modify volumeattributesclass: %v", err)
+			}
+
+			// Step 6: Verify new allowHostNQNs after update
+			ginkgo.By("Verifying updated allowHostNQNs configuration")
+			err = verifyAllowHostNQNsViaConnect(f, gatewayIP, hostListParam2["allowHostNQNs"])
+			Expect(err).ShouldNot(HaveOccurred())
+
+			// Step 7: Delete PVC
+			ginkgo.By("Deleting PVC with VAC")
+			err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			// validate created backend rbd images are deleted
+			validateRBDImageCount(f, 0, nvmeofPool)
+			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
+		})
 	})
 })

e2e/nvmeof_helper.go

@@ -17,10 +17,14 @@ limitations under the License.
 package e2e
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"sync"
 
 	"github.com/google/uuid"
+	"go.yaml.in/yaml/v2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -410,3 +414,150 @@ func mixedCreateDeletePodsOnly(
 
 	return nil
 }
+
+// nvmeHost represents a host entry from 'nvme list-subsys -o json'.
+// We only need HostNQN for verification; other fields are ignored during unmarshaling.
+type nvmeHost struct {
+	HostNQN string `json:"HostNQN"`
+}
+
+// verifyAllowHostNQNsViaConnect verifies that the allowHostNQNs configuration is working
+// by attempting to connect with allowed and disallowed host NQNs.
+func verifyAllowHostNQNsViaConnect(f *framework.Framework, gatewayIP, allowHostNQNsYAML string) error {
+	// Parse the allowed host NQNs from YAML format
+	allowedHosts := parseHostNQNsFromYAML(allowHostNQNsYAML)
+	if len(allowedHosts) == 0 {
+		return fmt.Errorf("no allowed hosts found in allowHostNQNs")
+	}
+
+	framework.Logf("Testing allowHostNQNs with %d allowed hosts", len(allowedHosts))
+
+	// Get a node to run commands on
+	nodes, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to list nodes: %w", err)
+	}
+	if len(nodes.Items) == 0 {
+		return fmt.Errorf("no nodes available")
+	}
+	nodeName := nodes.Items[0].Name
+
+	// Get the node-plugin pod on that node
+	nodePluginPod, err := getDaemonsetPodOnNode(f, nvmeofDaemonsetName, nodeName, cephCSINamespace)
+	if err != nil {
+		return fmt.Errorf("failed to get node-plugin pod: %w", err)
+	}
+
+	// Test 1: Try to connect with an allowed host NQN
+	allowedHost := allowedHosts[0]
+	framework.Logf("Testing connection with ALLOWED host NQN: %s", allowedHost)
+
+	connected := testNVMeConnection(f, nodePluginPod, gatewayIP, allowedHost)
+
+	if !connected {
+		return fmt.Errorf("connection failed for allowed host NQN %s", allowedHost)
+	}
+	framework.Logf("SUCCESS: Connected with allowed host NQN %s", allowedHost)
+
+	return nil
+}
+
+// testNVMeConnection attempts to connect to an NVMe-oF gateway with a specific host NQN
+// and verifies the connection via nvme list-subsys.
+func testNVMeConnection(f *framework.Framework, podName, gatewayIP, hostNQN string) bool {
+	// Ensure cleanup happens regardless of success or failure
+	defer disconnectAllNVMe(f, podName)
+
+	// Step 1: Try to connect with the specified host NQN
+	connectCmd := fmt.Sprintf("nvme connect-all --transport=tcp --traddr=%s --hostnqn=%s 2>&1", gatewayIP, hostNQN)
+
+	framework.Logf("Executing: %s", connectCmd)
+	stdout, stderr, err := execCommandInContainerByPodName(
+		f,
+		connectCmd,
+		cephCSINamespace,
+		podName,
+		nvmeofContainerName,
+	)
+
+	// Log the output
+	if stdout != "" {
+		framework.Logf("nvme connect-all stdout: %s", stdout)
+	}
+	if stderr != "" {
+		framework.Logf("nvme connect-all stderr: %s", stderr)
+	}
+
+	// Check if connection failed (expected for disallowed hosts)
+	if err != nil {
+		framework.Logf("Connection failed (expected for disallowed hosts): %v", err)
+
+		return false
+	}
+
+	// Step 2: Verify the connection using nvme list-subsys with JSON output
+	listCmd := "nvme list-subsys -o json 2>&1"
+	framework.Logf("Verifying connection with: %s", listCmd)
+
+	stdout, stderr, err = execCommandInContainerByPodName(
+		f,
+		listCmd,
+		cephCSINamespace,
+		podName,
+		nvmeofContainerName,
+	)
+
+	if err != nil {
+		framework.Logf("nvme list-subsys failed: %v, stderr: %s", err, stderr)
+
+		return false
+	}
+
+	// Step 3: Parse JSON output - it's an array of hosts
+	var hosts []nvmeHost
+	if err := json.Unmarshal([]byte(stdout), &hosts); err != nil {
+		framework.Logf("Failed to parse nvme list-subsys JSON output: %v", err)
+		framework.Logf("Raw output: %s", stdout)
+
+		return false
+	}
+
+	// Step 4: Check if the host with our NQN exists in the output
+	for _, host := range hosts {
+		if host.HostNQN == hostNQN {
+			framework.Logf("Found host in list-subsys: %s", host.HostNQN)
+
+			return true
+		}
+	}
+	framework.Logf("Host %s not found in list-subsys output", hostNQN)
+
+	return false
+}
+
+// disconnectAllNVMe disconnects all NVMe-oF connections.
+func disconnectAllNVMe(f *framework.Framework, podName string) {
+	disconnectCmd := "nvme disconnect-all 2>&1 || true"
+	stdout, stderr, err := execCommandInContainerByPodName(
+		f,
+		disconnectCmd,
+		cephCSINamespace,
+		podName,
+		nvmeofContainerName,
+	)
+	if err != nil {
+		framework.Logf("nvme disconnect-all warning: %v, stdout: %s, stderr: %s", err, stdout, stderr)
+	} else {
+		framework.Logf("Disconnected all NVMe connections")
+	}
+}
+
+// parseHostNQNsFromYAML parses the allowHostNQNs YAML format into a slice of NQNs.
+func parseHostNQNsFromYAML(yamlStr string) []string {
+	var hosts []string
+	if err := yaml.Unmarshal([]byte(yamlStr), &hosts); err != nil {
+		framework.Logf("Failed to parse allowHostNQNs YAML: %v", err)
+		return nil
+	}
+	return hosts
+}