csi: add 'CSIMetadataRadosNamespace' parameter to CephFilesystemSubVolumeGroup

This parameter is written to .spec.cephFS.radosNamespace of the generated
ClientProfile.

The parameter is called 'CSIMetadataRadosNamespace' to clarify that it is used
for CSI metadata and unrelated to the actual CephFS (meta)data.

Context:

The ceph-csi CephFS plugin stores additional metadata related to PV(C)s
in RADOS objects in the metadata pool of the CephFS.
By default those objects are stored in the 'csi' RADOS namespace.

For a multi-tenant CephFS setup (that is used by multiple K8s Clusters/
ceph-csi drivers) segregating those additional objects into separate RADOS
namespaces is desireable so the clusters can not modify the metadata of other
clusters (enforced by appropriate ceph client capabilities).

'ceph-csi' implemented this via the `cephFS.radosNamespace` config option for
cluster entries in the `ceph-csi-config` ConfigMap with ceph/ceph-csi#4661.

The `ceph-csi-operator` added support for that config entry by adding
'radosNamespace' to the 'CephFsConfigSpec' of the 'ClientProfile' with
ceph/ceph-csi-operator#165.

Signed-off-by: Alexander Degenhart <degenhart@fim.uni-passau.de>

Author: ein-stein-chen

Documentation/CRDs/specification.md

@@ -1575,6 +1575,19 @@ It must be unique among all Ceph clusters managed by Rook.
 If not specified, the clusterID will be generated and can be found in the CR status.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>csiMetadataRadosNamespace</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The RADOS namespace ceph-csi uses for additional metadata it stores in the metadata pool of the CephFS.
+If not specified the default of the ceph-csi driver is used.</p>
+</td>
+</tr>
 </table>
 </td>
 </tr>
@@ -4318,6 +4331,19 @@ It must be unique among all Ceph clusters managed by Rook.
 If not specified, the clusterID will be generated and can be found in the CR status.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>csiMetadataRadosNamespace</code><br/>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The RADOS namespace ceph-csi uses for additional metadata it stores in the metadata pool of the CephFS.
+If not specified the default of the ceph-csi driver is used.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.CephFilesystemSubVolumeGroupSpecPinning">CephFilesystemSubVolumeGroupSpecPinning

deploy/charts/rook-ceph/templates/resources.yaml

@@ -9367,6 +9367,16 @@ spec:
                   x-kubernetes-validations:
                     - message: ClusterID is immutable
                       rule: self == oldSelf
+                csiMetadataRadosNamespace:
+                  description: |-
+                    The RADOS namespace ceph-csi uses for additional metadata it stores in the metadata pool of the CephFS.
+                    If not specified the default of the ceph-csi driver is used.
+                  maxLength: 1024
+                  minLength: 1
+                  type: string
+                  x-kubernetes-validations:
+                    - message: CSIMetadataRadosNamespace is immutable
+                      rule: self == oldSelf
                 dataPoolName:
                   description: The data pool name for the Ceph Filesystem subvolume group layout, if the default CephFS pool is not desired.
                   type: string

deploy/examples/crds.yaml

@@ -9361,6 +9361,16 @@ spec:
                   x-kubernetes-validations:
                     - message: ClusterID is immutable
                       rule: self == oldSelf
+                csiMetadataRadosNamespace:
+                  description: |-
+                    The RADOS namespace ceph-csi uses for additional metadata it stores in the metadata pool of the CephFS.
+                    If not specified the default of the ceph-csi driver is used.
+                  maxLength: 1024
+                  minLength: 1
+                  type: string
+                  x-kubernetes-validations:
+                    - message: CSIMetadataRadosNamespace is immutable
+                      rule: self == oldSelf
                 dataPoolName:
                   description: The data pool name for the Ceph Filesystem subvolume group layout, if the default CephFS pool is not desired.
                   type: string

pkg/apis/ceph.rook.io/v1/types.go

@@ -3885,6 +3885,13 @@ type CephFilesystemSubVolumeGroupSpec struct {
 	// +kubebuilder:validation:MaxLength=36
 	// +kubebuilder:validation:Pattern=`^[a-zA-Z0-9_-]+$`
 	ClusterID string `json:"clusterID,omitempty"`
+	// The RADOS namespace ceph-csi uses for additional metadata it stores in the metadata pool of the CephFS.
+	// If not specified the default of the ceph-csi driver is used.
+	// +optional
+	// +kubebuilder:validation:XValidation:message="CSIMetadataRadosNamespace is immutable",rule="self == oldSelf"
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=1024
+	CSIMetadataRadosNamespace string `json:"csiMetadataRadosNamespace,omitempty"`
 }
 
 // CephFilesystemSubVolumeGroupSpecPinning represents the pinning configuration of SubVolumeGroup

pkg/operator/ceph/csi/config.go

@@ -58,15 +58,15 @@ func CreateUpdateClientProfileRadosNamespace(ctx context.Context, c client.Clien
 	return createUpdateClientProfile(c, clusterInfo, csiOpClientProfile)
 }
 
-func CreateUpdateClientProfileSubVolumeGroup(ctx context.Context, c client.Client, clusterInfo *cephclient.ClusterInfo, cephFilesystemSubVolumeGroupName, clusterID string) error {
+func CreateUpdateClientProfileSubVolumeGroup(ctx context.Context, c client.Client, clusterInfo *cephclient.ClusterInfo, cephFilesystemSubVolumeGroupName, clusterID string, csiMetadataRadosNamespace string) error {
 	logger.Info("Creating ceph-csi clientProfile CR for subvolume group")
 
-	csiOpClientProfile := generateProfileSubVolumeGroupSpec(clusterInfo, cephFilesystemSubVolumeGroupName, clusterID)
+	csiOpClientProfile := generateProfileSubVolumeGroupSpec(clusterInfo, cephFilesystemSubVolumeGroupName, clusterID, csiMetadataRadosNamespace)
 
 	return createUpdateClientProfile(c, clusterInfo, csiOpClientProfile)
 }
 
-func generateProfileSubVolumeGroupSpec(clusterInfo *cephclient.ClusterInfo, cephFilesystemSubVolumeGroupName, clusterID string) *csiopv1.ClientProfile {
+func generateProfileSubVolumeGroupSpec(clusterInfo *cephclient.ClusterInfo, cephFilesystemSubVolumeGroupName, clusterID string, csiMetadataRadosNamespace string) *csiopv1.ClientProfile {
 	csiOpClientProfile := &csiopv1.ClientProfile{}
 	csiOpClientProfile.Name = clusterID
 	csiOpClientProfile.Namespace = os.Getenv(k8sutil.PodNamespaceEnvVar)
@@ -76,6 +76,7 @@ func generateProfileSubVolumeGroupSpec(clusterInfo *cephclient.ClusterInfo, ceph
 		},
 		CephFs: &csiopv1.CephFsConfigSpec{
 			SubVolumeGroup: cephFilesystemSubVolumeGroupName,
+			RadosNamespace: &csiMetadataRadosNamespace,
 			CephCsiSecrets: &csiopv1.CephCsiSecretsSpec{
 				ControllerPublishSecret: v1.SecretReference{
 					Name:      CsiCephFSProvisionerSecret,

pkg/operator/ceph/csi/config_test.go

@@ -55,6 +55,7 @@ func TestCreateUpdateClientProfile(t *testing.T) {
 
 	cephBlockPoolRadosNamespacedName := types.NamespacedName{Namespace: ns, Name: "cephBlockPoolRadosNames"}
 	cephSubVolGrpNamespacedName := types.NamespacedName{Namespace: ns, Name: "cephSubVolumeGroupNames"}
+	cephSubVolGrpRadosNamespaceNamespacedName := types.NamespacedName{Namespace: ns, Name: "radosNamespaceName"}
 	csiOpClientProfile := &csiopv1.ClientProfile{}
 
 	// Register operator types with the runtime scheme.
@@ -69,7 +70,7 @@ func TestCreateUpdateClientProfile(t *testing.T) {
 	err := CreateUpdateClientProfileRadosNamespace(context.TODO(), cl, c, cephBlockPoolRadosNamespacedName.Name, cephBlockPoolRadosNamespacedName.Name)
 	assert.NoError(t, err)
 
-	err = CreateUpdateClientProfileSubVolumeGroup(context.TODO(), cl, c, cephSubVolGrpNamespacedName.Name, cephSubVolGrpNamespacedName.Name)
+	err = CreateUpdateClientProfileSubVolumeGroup(context.TODO(), cl, c, cephSubVolGrpNamespacedName.Name, cephSubVolGrpNamespacedName.Name, cephSubVolGrpRadosNamespaceNamespacedName.Name)
 	assert.NoError(t, err)
 
 	err = cl.Get(context.TODO(), cephBlockPoolRadosNamespacedName, csiOpClientProfile)
@@ -80,6 +81,7 @@ func TestCreateUpdateClientProfile(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, csiOpClientProfile.Spec.CephFs.SubVolumeGroup, cephSubVolGrpNamespacedName.Name)
 	assert.Equal(t, csiOpClientProfile.Spec.CephFs.KernelMountOptions["ms_mode"], kernelMountKeyVal[1])
+	assert.Equal(t, *csiOpClientProfile.Spec.CephFs.RadosNamespace, cephSubVolGrpRadosNamespaceNamespacedName.Name)
 }
 
 func TestParseMountOptions(t *testing.T) {

pkg/operator/ceph/file/subvolumegroup/controller.go

@@ -281,7 +281,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
 		}
 		r.updateStatus(observedGeneration, namespacedName, cephv1.ConditionReady)
 		if csi.EnableCSIOperator() {
-			err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup))
+			err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup), cephFilesystemSubVolumeGroup.Spec.CSIMetadataRadosNamespace)
 			if err != nil {
 				return reconcile.Result{}, errors.Wrap(err, "failed to create ceph csi-op config CR for subvolume")
 			}
@@ -334,7 +334,7 @@ func (r *ReconcileCephFilesystemSubVolumeGroup) reconcile(request reconcile.Requ
 	r.updateStatus(observedGeneration, request.NamespacedName, cephv1.ConditionReady)
 
 	if csi.EnableCSIOperator() {
-		err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup))
+		err = csi.CreateUpdateClientProfileSubVolumeGroup(r.clusterInfo.Context, r.client, r.clusterInfo, cephFilesystemSubVolumeGroupName, buildClusterID(cephFilesystemSubVolumeGroup), cephFilesystemSubVolumeGroup.Spec.CSIMetadataRadosNamespace)
 		if err != nil {
 			return reconcile.Result{}, errors.Wrap(err, "failed to create ceph csi-op config CR for subvolumeGroup")
 		}