e2e: add VolumeAttributesClass example and e2e test

Add a VolumeAttributesClass example for NVMe-oF volumes
demonstrating the mutable parameters supported by the driver:
- Host access control via allowHostNQNs parameter (for external hosts)
- QoS parameters

The example includes a working configuration
with host access control enabled,
and all optional parameters documented
as comments with explanations.

Also add an e2e test that validates VAC functionality by:
1. Creating a VAC with allowHostNQNs parameter
2. Creating a PVC that references the VAC
3. Verifying volume creation
4. Validating cleanup

Signed-off-by: gadi-didi <gadi.didi@ibm.com>

Author: gadididi

e2e/nvmeof-deploy.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"strings"
 	"time"
 
@@ -29,6 +30,7 @@ import (
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -242,3 +244,73 @@ func createNVMeoFCredentials(f *framework.Framework) {
 	err = createRBDSecret(f, nvmeofNodePluginSecretName, nvmeofKeyringNodePluginUsername, key)
 	Expect(err).ShouldNot(HaveOccurred())
 }
+
+func createNVMeOFVolumeAttributesClass(
+	c kubernetes.Interface,
+	name string,
+	params map[string]string,
+) error {
+	vac, err := getVolumeAttributesClass(nvmeofExamplePath + "volumeattributesclass.yaml")
+	if err != nil {
+		return fmt.Errorf("failed to get vac: %w", err)
+	}
+	if name != "" {
+		vac.Name = name
+	}
+
+	// overload any parameters that were passed
+	if params == nil {
+		// create an empty params, so that params["clusterID"] below
+		// does not panic
+		params = map[string]string{}
+	}
+	maps.Copy(vac.Parameters, params)
+
+	timeout := time.Duration(deployTimeout) * time.Minute
+
+	return wait.PollUntilContextTimeout(context.TODO(), poll, timeout, true, func(ctx context.Context) (bool, error) {
+		_, err = c.StorageV1().VolumeAttributesClasses().Create(ctx, &vac, metav1.CreateOptions{})
+		if err != nil {
+			if apierrs.IsAlreadyExists(err) {
+				return true, nil
+			}
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			framework.Logf("error creating VolumeAttributesClass %q: %v", vac.Name, err)
+
+			return false, fmt.Errorf("failed to create VolumeAttributesClass %q: %w", vac.Name, err)
+		}
+
+		return true, nil
+	})
+}
+
+func deleteNVMeOFVolumeAttributesClass(c kubernetes.Interface, name string) error {
+	vac, err := getVolumeAttributesClass(nvmeofExamplePath + "volumeattributesclass.yaml")
+	if err != nil {
+		return err
+	}
+	if name != "" {
+		vac.Name = name
+	}
+
+	timeout := time.Duration(deployTimeout) * time.Minute
+
+	return wait.PollUntilContextTimeout(context.TODO(), poll, timeout, true, func(ctx context.Context) (bool, error) {
+		err = c.StorageV1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})
+		if err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			framework.Logf("error deleting VolumeAttributesClass %q: %v", vac.Name, err)
+
+			return false, fmt.Errorf("failed to delete VolumeAttributesClass %q: %w", vac.Name, err)
+		}
+
+		return true, nil
+	})
+}

e2e/nvmeof.go

@@ -311,5 +311,100 @@ var _ = ginkgo.Describe("nvmeof", func() {
 			framework.Logf("GroupLock Pods-only test passed: %d Pods created and deleted with rapid Group A/B switching",
 				totalCount)
 		})
+
+		ginkgo.It("create a PVC with VAC containing allowHostNQNs, apply new VAC then delete the PVC", func() {
+
+			// Vac test requires Kubernetes 1.34+ for VolumeAttributesClass support
+			if !k8sVersionGreaterEquals(f.ClientSet, 1, 34) {
+				framework.Logf("skipping VolumeAttributesClass test, needs Kubernetes >= 1.34")
+
+				return
+			}
+
+			// This test validates VolumeAttributesClass (VAC) support for NVMe-oF volumes
+			// with host access control via the allowHostNQNs parameter.
+			//
+			// Test flow:
+			// 1. Create a VAC with allowHostNQNs parameter containing a list of allowed host NQNs
+			// 2. Create a PVC that references the VAC via spec.volumeAttributesClassName
+			// 3. Verify the volume is created and the host access control is applied
+			// 4. Update the PVC to reference a different VAC and verify the update is applied
+			// 5. Delete the PVC and verify cleanup
+			//
+			// This tests the ControllerModifyVolume functionality in the NVMe-oF driver,
+			// ensuring that host access lists can be configured via VAC at volume creation time.
+
+			// Step 1: Create 2 VACs with allowHostNQNs parameter
+			vacName1 := "e2e-" + f.UniqueName + "-vac1"
+			vacName2 := "e2e-" + f.UniqueName + "-vac2"
+			hostListParam := map[string]string{
+				"allowHostNQNs": `- nqn.2014-08.org.nvmexpress:test-host-1
+- nqn.2014-08.org.nvmexpress:test-host-2
+- nqn.2014-08.org.nvmexpress:test-host-3`,
+			}
+			hostListParam2 := map[string]string{
+				"allowHostNQNs": `- nqn.2014-08.org.nvmexpress:test-host-1
+- nqn.2014-08.org.nvmexpress:test-host-5`,
+			}
+			ginkgo.By("Creating VolumeAttributesClass with allowHostNQNs")
+			err := createNVMeOFVolumeAttributesClass(f.ClientSet, vacName1, hostListParam)
+			Expect(err).ShouldNot(HaveOccurred())
+			defer func() {
+				err = deleteNVMeOFVolumeAttributesClass(f.ClientSet, vacName1)
+				if err != nil {
+					logAndFail("failed to delete volumeattributesclass: %v", err)
+				}
+			}()
+
+			ginkgo.By("Creating VolumeAttributesClass with allowHostNQNs")
+			err = createNVMeOFVolumeAttributesClass(f.ClientSet, vacName2, hostListParam2)
+			Expect(err).ShouldNot(HaveOccurred())
+			defer func() {
+				err = deleteNVMeOFVolumeAttributesClass(f.ClientSet, vacName2)
+				if err != nil {
+					logAndFail("failed to delete volumeattributesclass: %v", err)
+				}
+			}()
+
+			// Step 2: Create PVC with reference to the VAC1
+			pvc, err := loadPVC(pvcPath)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			pvc.Namespace = f.UniqueName
+			pvc.Spec.StorageClassName = &nvmeofStorageClass
+			pvc.Spec.VolumeAttributesClassName = &vacName1
+
+			ginkgo.By("Creating PVC with VolumeAttributesClass reference")
+			err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			pv, err := getBoundPV(f.ClientSet, pvc)
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(pv.Spec.VolumeAttributesClassName).NotTo(BeNil())
+			Expect(*pv.Spec.VolumeAttributesClassName).To(Equal(vacName1))
+
+			validateRBDImageCount(f, 1, nvmeofPool)
+			validateOmapCount(f, 1, rbdType, nvmeofPool, volumesType)
+
+			// Step 3: Update PVC to reference VAC2 and verify the update is applied
+			ginkgo.By("Updating PVC to reference a different VolumeAttributesClass")
+
+			err = modifyPVCVolumeAttributesClass(
+				f.ClientSet,
+				pvc,
+				vacName2)
+			if err != nil {
+				logAndFail("failed to modify volumeattributesclass: %v", err)
+			}
+
+			// Step 3: Delete PVC
+			ginkgo.By("Deleting PVC with VAC")
+			err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			// validate created backend rbd images are deleted
+			validateRBDImageCount(f, 0, nvmeofPool)
+			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
+		})
 	})
 })

examples/nvmeof/volumeattributesclass.yaml

@@ -0,0 +1,48 @@
+---
+# VolumeAttributesClass for NVMe-oF volumes
+# This example demonstrates optional mutable parameters for NVMe-oF
+apiVersion: storage.k8s.io/v1
+kind: VolumeAttributesClass
+metadata:
+  name: nvmeof-vac
+driverName: nvmeof.csi.ceph.com
+parameters:
+  # Host access control: YAML list of host NQNs allowed to access this volume
+  # Format: nqn.<date>.<reverse-domain>:<user-part>
+  # For Ceph-CSI managed hosts: nqn.2014-08.org.nvmexpress:<nodeID>
+  allowHostNQNs: |
+    - nqn.2014-08.org.nvmexpress:host1
+    - nqn.2014-08.org.nvmexpress:host2
+    - nqn.2014-08.org.nvmexpress:node-123
+
+  # Alternative: Allow any host to access the volume
+  # allowHostNQNs: |
+  #   - "*"
+
+  # Alternative: Empty list denies all hosts
+  # allowHostNQNs: |
+
+  # Optional QoS parameters (uint64 values):
+
+  # Limit read/write IOPS (operations per second for both read and write)
+  # nvmeofRWIOsPerSecond: "10000"
+
+  # Limit read/write throughput in MB/s (for both read and write)
+  # nvmeofRWMegabytesPerSecond: "100"
+
+  # Limit read-only IOPS (operations per second for read only)
+  # nvmeofRIOsPerSecond: "5000"
+
+  # Limit write-only IOPS (operations per second for write only)
+  # nvmeofWIOsPerSecond: "5000"
+
+  # Limit read-only throughput in MB/s
+  # nvmeofRMegabytesPerSecond: "50"
+
+  # Limit write-only throughput in MB/s
+  # nvmeofWMegabytesPerSecond: "50"
+
+# Notes:
+# - Cannot mix RBD QoS parameters with NVMe-oF QoS parameters
+# - VolumeAttributesClass can be modified at runtime using kubectl
+# - Changes are applied dynamically via ControllerModifyVolume