diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 2cce5d928e6..1e7fd32510b 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -21,5 +21,6 @@ - nvmeof: add Kubernetes ServiceAccount based volume access restriction - cephfs: add Kubernetes ServiceAccount based volume access restriction - nfs: add Kubernetes ServiceAccount based volume access restriction +- nfs: add snapshot-backed (shallow) read-only volume support ## NOTE diff --git a/e2e/nfs.go b/e2e/nfs.go index 2f7697d9d0f..0c9c2712f5a 100644 --- a/e2e/nfs.go +++ b/e2e/nfs.go @@ -1249,6 +1249,471 @@ var _ = Describe("nfs", func() { validateOmapCount(f, 0, cephfsType, metadataPool, volumesType) }) + It("checking snapshot-backed volume", func() { + err := createNFSSnapshotClass(f) + if err != nil { + logAndFail("failed to create NFS snapshotclass: %v", err) + } + defer func() { + err = deleteNFSSnapshotClass() + if err != nil { + logAndFail("failed to delete VolumeSnapshotClass: %v", err) + } + }() + + err = createNFSStorageClass(f.ClientSet, f, false, nil) + if err != nil { + logAndFail("failed to create NFS storageclass: %v", err) + } + + pvc, err := loadPVC(pvcPath) + if err != nil { + logAndFail("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + logAndFail("failed to create PVC: %v", err) + } + + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + logAndFail("failed to get PV object for %s: %v", pvc.Name, err) + } + + app, err := loadApp(appPath) + if err != nil { + logAndFail("failed to load application: %v", err) + } + app.Namespace = f.UniqueName + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + appLabels := map[string]string{ + appKey: appLabel, + } + app.Labels = appLabels + optApp := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, appLabels[appKey]), + } + err = writeDataInPod(app, &optApp, f) + if err != nil { + logAndFail("failed to write data: %v", err) + } + + appTestFilePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + err = createSnapshot(&snap, deployTimeout) + if err != nil { + logAndFail("failed to create snapshot: %v", err) + } + validateCephFSSnapshotCount(f, 1, defaultSubvolumegroup, pv) + + err = appendToFileInContainer(f, app, appTestFilePath, "hello", &optApp) + if err != nil { + logAndFail("failed to append data: %v", err) + } + + parentFileSum, err := calculateSHA512sum(f, app, appTestFilePath, &optApp) + if err != nil { + logAndFail("failed to get SHA512 sum for file: %v", err) + } + + pvcClone, err := loadPVC(pvcClonePath) + if err != nil { + logAndFail("failed to load PVC: %v", err) + } + // Snapshot-backed volumes support read-only access modes only. + pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + appClone, err := loadApp(appClonePath) + if err != nil { + logAndFail("failed to load application: %v", err) + } + appCloneLabels := map[string]string{ + appKey: appCloneLabel, + } + appClone.Labels = appCloneLabels + optAppClone := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, appCloneLabels[appKey]), + } + pvcClone.Namespace = f.UniqueName + appClone.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + logAndFail("failed to create PVC and app: %v", err) + } + + // Snapshot-backed volume shouldn't contribute to total subvolume count. + validateSubvolumeCount(f, 1, fileSystemName, defaultSubvolumegroup) + + // Deleting snapshot before deleting pvcClone should succeed. It will be + // deleted once all volumes that are backed by this snapshot are gone. + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + logAndFail("failed to delete snapshot: %v", err) + } + + appCloneTestFilePath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + + snapFileSum, err := calculateSHA512sum(f, appClone, appCloneTestFilePath, &optAppClone) + if err != nil { + logAndFail("failed to get SHA512 sum for file: %v", err) + } + + if parentFileSum == snapFileSum { + logAndFail("SHA512 sums of files in parent subvol and snapshot should differ") + } + + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + logAndFail("failed to delete PVC or application: %v", err) + } + + validateCephFSSnapshotCount(f, 0, defaultSubvolumegroup, pv) + + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + logAndFail("failed to delete PVC or application: %v", err) + } + + err = deleteResource(nfsExamplePath + "storageclass.yaml") + if err != nil { + logAndFail("failed to delete NFS storageclass: %v", err) + } + + err = createNFSStorageClass(f.ClientSet, f, false, nil) + if err != nil { + logAndFail("failed to create NFS storageclass: %v", err) + } + }) + + It("checking snapshot-backed volume by backing snapshot as false", func() { + err := createNFSStorageClass(f.ClientSet, f, false, nil) + if err != nil { + logAndFail("failed to create NFS storageclass: %v", err) + } + + pvc, err := loadPVC(pvcPath) + if err != nil { + logAndFail("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + logAndFail("failed to create PVC: %v", err) + } + + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + logAndFail("failed to get PV object for %s: %v", pvc.Name, err) + } + + app, err := loadApp(appPath) + if err != nil { + logAndFail("failed to load application: %v", err) + } + app.Namespace = f.UniqueName + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + appLabels := map[string]string{ + appKey: appLabel, + } + app.Labels = appLabels + optApp := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, appLabels[appKey]), + } + err = writeDataInPod(app, &optApp, f) + if err != nil { + logAndFail("failed to write data: %v", err) + } + + appTestFilePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + + err = createNFSSnapshotClass(f) + if err != nil { + logAndFail("failed to create NFS snapshotclass: %v", err) + } + defer func() { + err = deleteNFSSnapshotClass() + if err != nil { + logAndFail("failed to delete VolumeSnapshotClass: %v", err) + } + }() + + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + err = createSnapshot(&snap, deployTimeout) + if err != nil { + logAndFail("failed to create snapshot: %v", err) + } + validateCephFSSnapshotCount(f, 1, defaultSubvolumegroup, pv) + + err = appendToFileInContainer(f, app, appTestFilePath, "hello", &optApp) + if err != nil { + logAndFail("failed to append data: %v", err) + } + + parentFileSum, err := calculateSHA512sum(f, app, appTestFilePath, &optApp) + if err != nil { + logAndFail("failed to get SHA512 sum for file: %v", err) + } + + err = deleteResource(nfsExamplePath + "storageclass.yaml") + if err != nil { + logAndFail("failed to delete NFS storageclass: %v", err) + } + + err = createNFSStorageClass(f.ClientSet, f, false, map[string]string{ + "backingSnapshot": "false", + }) + if err != nil { + logAndFail("failed to create NFS storageclass: %v", err) + } + + pvcClone, err := loadPVC(pvcClonePath) + if err != nil { + logAndFail("failed to load PVC: %v", err) + } + // Snapshot-backed volumes support read-only access modes only. + pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + appClone, err := loadApp(appClonePath) + if err != nil { + logAndFail("failed to load application: %v", err) + } + appCloneLabels := map[string]string{ + appKey: appCloneLabel, + } + appClone.Labels = appCloneLabels + optAppClone := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, appCloneLabels[appKey]), + } + pvcClone.Namespace = f.UniqueName + appClone.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + logAndFail("failed to create PVC and app: %v", err) + } + + validateSubvolumeCount(f, 2, fileSystemName, defaultSubvolumegroup) + + // Deleting snapshot before deleting pvcClone should succeed. It will be + // deleted once all volumes that are backed by this snapshot are gone. + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + logAndFail("failed to delete snapshot: %v", err) + } + validateCephFSSnapshotCount(f, 0, defaultSubvolumegroup, pv) + + appCloneTestFilePath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + + snapFileSum, err := calculateSHA512sum(f, appClone, appCloneTestFilePath, &optAppClone) + if err != nil { + logAndFail("failed to get SHA512 sum for file: %v", err) + } + + if parentFileSum == snapFileSum { + logAndFail("SHA512 sums of files in parent subvol and snapshot should differ") + } + + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + logAndFail("failed to delete PVC or application: %v", err) + } + + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + logAndFail("failed to delete PVC or application: %v", err) + } + + err = deleteResource(nfsExamplePath + "storageclass.yaml") + if err != nil { + logAndFail("failed to delete NFS storageclass: %v", err) + } + + err = createNFSStorageClass(f.ClientSet, f, false, nil) + if err != nil { + logAndFail("failed to create NFS storageclass: %v", err) + } + }) + + It("create RWX clone from ROX PVC", func() { + err := createNFSStorageClass(f.ClientSet, f, false, nil) + if err != nil { + logAndFail("failed to create NFS storageclass: %v", err) + } + + pvc, err := loadPVC(pvcPath) + if err != nil { + logAndFail("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + logAndFail("failed to create PVC: %v", err) + } + + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + logAndFail("failed to get PV object for %s: %v", pvc.Name, err) + } + + app, err := loadApp(appPath) + if err != nil { + logAndFail("failed to load application: %v", err) + } + app.Namespace = f.UniqueName + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + appLabels := map[string]string{ + appKey: appLabel, + } + app.Labels = appLabels + optApp := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, appLabels[appKey]), + } + err = writeDataInPod(app, &optApp, f) + if err != nil { + logAndFail("failed to write data: %v", err) + } + + appTestFilePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + + err = appendToFileInContainer(f, app, appTestFilePath, "hello", &optApp) + if err != nil { + logAndFail("failed to append data: %v", err) + } + + parentFileSum, err := calculateSHA512sum(f, app, appTestFilePath, &optApp) + if err != nil { + logAndFail("failed to get SHA512 sum for file: %v", err) + } + + err = createNFSSnapshotClass(f) + if err != nil { + logAndFail("failed to create NFS snapshotclass: %v", err) + } + defer func() { + err = deleteNFSSnapshotClass() + if err != nil { + logAndFail("failed to delete VolumeSnapshotClass: %v", err) + } + }() + + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + err = createSnapshot(&snap, deployTimeout) + if err != nil { + logAndFail("failed to create snapshot: %v", err) + } + validateCephFSSnapshotCount(f, 1, defaultSubvolumegroup, pv) + + pvcClone, err := loadPVC(pvcClonePath) + if err != nil { + logAndFail("failed to load PVC: %v", err) + } + // Snapshot-backed volumes support read-only access modes only. + pvcClone.Spec.DataSource.Name = snap.Name + pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + + pvcClone.Namespace = f.UniqueName + err = createPVCAndvalidatePV(c, pvcClone, deployTimeout) + if err != nil { + logAndFail("failed to create PVC: %v", err) + } + + validateSubvolumeCount(f, 1, fileSystemName, defaultSubvolumegroup) + + // create RWX clone from ROX PVC + pvcRWXClone, err := loadPVC(pvcSmartClonePath) + if err != nil { + logAndFail("failed to load PVC: %v", err) + } + pvcRWXClone.Spec.DataSource.Name = pvcClone.Name + pvcRWXClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany} + pvcRWXClone.Namespace = f.UniqueName + + appClone, err := loadApp(appPath) + if err != nil { + logAndFail("failed to load application: %v", err) + } + appCloneLabels := map[string]string{ + appKey: appCloneLabel, + } + appClone.Name = f.UniqueName + "-app" + appClone.Namespace = f.UniqueName + appClone.Labels = appCloneLabels + appClone.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcRWXClone.Name + optAppClone := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, appCloneLabels[appKey]), + } + + err = createPVCAndApp("", f, pvcRWXClone, appClone, deployTimeout) + if err != nil { + logAndFail("failed to create PVC and app: %v", err) + } + // 2 subvolumes should be created 1 for parent PVC and 1 for + // RWX clone PVC. + validateSubvolumeCount(f, 2, fileSystemName, defaultSubvolumegroup) + + appCloneTestFilePath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + + cloneFileSum, err := calculateSHA512sum(f, appClone, appCloneTestFilePath, &optAppClone) + if err != nil { + logAndFail("failed to get SHA512 sum for file: %v", err) + } + + if parentFileSum != cloneFileSum { + logAndFail( + "SHA512 sums of files in parent and ROX should not differ. parentFileSum: %s cloneFileSum: %s", + parentFileSum, + cloneFileSum) + } + + // Now try to write to the PVC as its a RWX PVC + err = appendToFileInContainer(f, app, appCloneTestFilePath, "testing", &optApp) + if err != nil { + logAndFail("failed to append data: %v", err) + } + + // Deleting snapshot before deleting pvcClone should succeed. It will be + // deleted once all volumes that are backed by this snapshot are gone. + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + logAndFail("failed to delete snapshot: %v", err) + } + + // delete parent pvc and app + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + logAndFail("failed to delete PVC or application: %v", err) + } + + // delete ROX clone PVC + err = deletePVCAndValidatePV(c, pvcClone, deployTimeout) + if err != nil { + logAndFail("failed to delete PVC or application: %v", err) + } + // delete RWX clone PVC and app + err = deletePVCAndApp("", f, pvcRWXClone, appClone) + if err != nil { + logAndFail("failed to delete PVC or application: %v", err) + } + + validateSubvolumeCount(f, 0, fileSystemName, defaultSubvolumegroup) + validateOmapCount(f, 0, cephfsType, metadataPool, volumesType) + + err = deleteResource(nfsExamplePath + "storageclass.yaml") + if err != nil { + logAndFail("failed to delete NFS storageclass: %v", err) + } + + err = createNFSStorageClass(f.ClientSet, f, false, nil) + if err != nil { + logAndFail("failed to create NFS storageclass: %v", err) + } + }) + It("delete NFS provisioner and plugin secret", func() { // delete nfs provisioner secret err := deleteCephUser(f, keyringCephFSProvisionerUsername) diff --git a/internal/cephfs/controllerserver.go b/internal/cephfs/controllerserver.go index c40b274bea3..0a08020bcac 100644 --- a/internal/cephfs/controllerserver.go +++ b/internal/cephfs/controllerserver.go @@ -272,6 +272,9 @@ func buildCreateVolumeResponse( volumeContext := util.GetVolumeContext(req.GetParameters()) volumeContext["subvolumeName"] = vID.FsSubvolName volumeContext["subvolumePath"] = volOptions.RootPath + if volOptions.BackingSnapshot { + volumeContext["backingSnapshotRoot"] = volOptions.BackingSnapshotRoot + } volume := &csi.Volume{ VolumeId: vID.VolumeID, CapacityBytes: volOptions.Size, diff --git a/internal/nfs/controller/controllerserver.go b/internal/nfs/controller/controllerserver.go index be8d55a8732..b7b4146ac25 100644 --- a/internal/nfs/controller/controllerserver.go +++ b/internal/nfs/controller/controllerserver.go @@ -19,6 +19,7 @@ package controller import ( "context" "errors" + "path" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -78,8 +79,6 @@ func (cs *Server) CreateVolume( ctx context.Context, req *csi.CreateVolumeRequest, ) (*csi.CreateVolumeResponse, error) { - // nfs does not supports shallow snapshots - req.Parameters["backingSnapshot"] = "false" res, err := cs.backendServer.CreateVolume(ctx, req) if err != nil { return nil, err @@ -87,6 +86,14 @@ func (cs *Server) CreateVolume( backend := res.GetVolume() + // For snapshot-backed volumes, adjust the subvolumePath to include + // the backing snapshot root, so the NFS export points directly to + // the snapshot data directory. + if bsr := backend.GetVolumeContext()["backingSnapshotRoot"]; bsr != "" { + backend.VolumeContext["subvolumePath"] = path.Join( + backend.GetVolumeContext()["subvolumePath"], bsr) + } + log.DebugLog(ctx, "CephFS volume created: %s", backend.GetVolumeId()) secret := req.GetSecrets()