Skip to content

Commit e5003f8

Browse files
nixpanicnixpanic
committed
e2e: add nvmeof driver testing support with k8s-e2e-external-storage
Add configuration files and scripts to enable NVMeoF driver testing in the k8s-e2e-external-storage CI job. This includes the driver manifest, StorageClass template, and VolumeSnapshotClass template for NVMeoF. The create-storageclasses.sh script now auto-detects the NVMeoF gateway address from the rook-ceph-nvmeof service, or it can be configured via the GATEWAY_ADDRESS and LISTENERS environment variables. Assisted-by: Claude Code <noreply@anthropic.com> Signed-off-by: Niels de Vos <ndevos@ibm.com>
1 parent 7ecdfb6 commit e5003f8

5 files changed

Lines changed: 174 additions & 2 deletions

File tree

‎scripts/k8s-storage/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,23 @@ The Ceph-CSI Configuration from the `ceph-csi-config` ConfigMap is created with
1717
referenced in the StorageClasses and contains the connection details for the
1818
Ceph cluster.
1919

20+
## Driver-specific Configuration
21+
22+
### NVMeoF Driver
23+
24+
The NVMeoF driver requires additional configuration parameters that can be set
25+
via environment variables before running `create-storageclasses.sh`:
26+
27+
- `GATEWAY_ADDRESS`: NVMeoF gateway IP address (auto-detected from
28+
`rook-ceph-nvmeof` service if available)
29+
- `LISTENERS`: JSON array of listener configurations (auto-generated if not set)
30+
31+
Example:
32+
33+
```bash
34+
export GATEWAY_ADDRESS=10.242.64.32
35+
export LISTENERS='[{"address": "10.242.64.32"}]'
36+
./create-storageclasses.sh
37+
```
38+
2039
[1]: https://github.com/kubernetes/kubernetes/tree/master/test/e2e/storage/external

‎scripts/k8s-storage/create-storageclasses.sh‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,37 @@ WORKDIR=$(dirname "${0}")
2020
TOOLBOX_POD=$(kubectl -n rook-ceph get pods --no-headers -l app=rook-ceph-tools -o=jsonpath='{.items[0].metadata.name}')
2121
FS_ID=$(kubectl -n rook-ceph exec "${TOOLBOX_POD}" -- ceph fsid)
2222

23+
# NVMeoF-specific parameters (can be overridden via environment variables)
24+
GATEWAY_ADDRESS=${GATEWAY_ADDRESS:-""}
25+
LISTENERS=${LISTENERS:-""}
26+
27+
# Auto-detect gateway address and listeners from rook-ceph-nvmeof service if not set
28+
if [ -z "${GATEWAY_ADDRESS}" ]; then
29+
GATEWAY_ADDRESS=$(kubectl -n rook-ceph get service -l app=rook-ceph-nvmeof -o=jsonpath='{.spec.clusterIP}' 2>/dev/null || echo "")
30+
fi
31+
32+
if [ -z "${LISTENERS}" ] && [ -n "${GATEWAY_ADDRESS}" ]; then
33+
# Create a simple listener config with the gateway address
34+
LISTENERS='[{"address": "'"${GATEWAY_ADDRESS}"'"}]'
35+
fi
36+
2337
for sc in "${WORKDIR}"/sc-*.yaml.in
2438
do
25-
sed "s/@@CLUSTER_ID@@/${FS_ID}/" "${sc}" |
26-
kubectl create -f -
39+
# Start with CLUSTER_ID replacement
40+
SC_CONTENT=$(sed "s/@@CLUSTER_ID@@/${FS_ID}/" "${sc}")
41+
42+
# For nvmeof, also replace nvmeof-specific parameters
43+
if echo "${sc}" | grep -q "nvmeof"; then
44+
if [ -z "${GATEWAY_ADDRESS}" ]; then
45+
echo "Warning: GATEWAY_ADDRESS not set and could not auto-detect rook-ceph-nvmeof service"
46+
echo "Skipping ${sc}"
47+
continue
48+
fi
49+
50+
SC_CONTENT=$(echo "${SC_CONTENT}" | sed \
51+
-e "s|@@GATEWAY_ADDRESS@@|${GATEWAY_ADDRESS}|g" \
52+
-e "s|@@LISTENERS@@|${LISTENERS}|g")
53+
fi
54+
55+
echo "${SC_CONTENT}" | kubectl create -f -
2756
done

‎scripts/k8s-storage/driver-nvmeof.yaml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
---
2+
ShortName: cephcsi-nvmeof-test
3+
Timeouts:
4+
ClaimProvision: 10m
5+
6+
StorageClass:
7+
FromExistingClassName: k8s-storage-e2e-nvmeof
8+
# FromFile: sc-nvmeof.yaml
9+
10+
SnapshotClass:
11+
# Must be set to enable snapshotting tests
12+
FromExistingClassName: k8s-storage-e2e-nvmeof
13+
14+
DriverInfo:
15+
# Internal name of the driver, display name in the test case and test objects
16+
Name: nvmeof.csi.ceph.com
17+
18+
# The range of disk size supported by this driver
19+
SupportedSizeRange:
20+
Min: 1Gi
21+
Max: 16Ti
22+
23+
# Map of strings for supported FS types
24+
SupportedFsType:
25+
ext4: {}
26+
xfs: {}
27+
28+
# Map of strings for supported mount options
29+
SupportedMountOption:
30+
rw: {}
31+
32+
# Map of strings for required mount options
33+
RequiredMountOption:
34+
rw: {}
35+
36+
# Optional list of access modes required for provisioning. Default is RWO
37+
# RequiredAccessModes:
38+
39+
# Map that represents the capabilities the driver supports
40+
Capabilities:
41+
# Data is persisted across pod restarts
42+
persistence: true
43+
44+
# Volume ownership via fsGroup
45+
fsGroup: false
46+
47+
# Raw block mode
48+
block: true
49+
50+
# Exec a file in the volume
51+
exec: true
52+
53+
# Support for volume limits
54+
volumeLimits: false
55+
56+
# Support for volume expansion in controllers
57+
controllerExpansion: true
58+
59+
# Support for volume expansion in nodes
60+
nodeExpansion: true
61+
62+
# supports offline volume expansion
63+
offlineExpansion: false
64+
65+
# Support volume that can run on single node only (like hostpath)
66+
singleNodeVolume: false
67+
68+
# Support ReadWriteMany access modes (block mode only)
69+
RWX: false
70+
71+
# Support topology
72+
topology: false
73+
74+
# Support populate data from snapshot
75+
snapshotDataSource: true
76+
77+
# Support populated data from PVC
78+
pvcDataSource: true
79+
80+
# multiple pods on a node can use the same volume concurrently
81+
multipods: true
82+
83+
# support ReadWriteOncePod access mode
84+
readWriteOncePod: true
85+
86+
# supports ROX AccessMode in PVC for PVC with Snapshot DataSource
87+
capReadOnlyMany: true

‎scripts/k8s-storage/sc-nvmeof.yaml.in‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
apiVersion: storage.k8s.io/v1
3+
kind: StorageClass
4+
metadata:
5+
name: k8s-storage-e2e-nvmeof
6+
provisioner: nvmeof.csi.ceph.com
7+
parameters:
8+
clusterID: @@CLUSTER_ID@@
9+
pool: replicapool
10+
imageFeatures: layering
11+
csi.storage.k8s.io/provisioner-secret-name: rook-csi-nvmeof-provisioner
12+
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
13+
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-nvmeof-provisioner
14+
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
15+
csi.storage.k8s.io/controller-publish-secret-name: rook-csi-nvmeof-node
16+
csi.storage.k8s.io/controller-publish-secret-namespace: rook-ceph
17+
csi.storage.k8s.io/node-stage-secret-name: rook-csi-nvmeof-node
18+
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
19+
csi.storage.k8s.io/fstype: ext4
20+
subsystemNQN: nqn.2025-08.io.ceph:k8s-ceph-csi-e2e
21+
nvmeofGatewayAddress: @@GATEWAY_ADDRESS@@
22+
listeners: @@LISTENERS@@
23+
reclaimPolicy: Delete
24+
allowVolumeExpansion: true
25+
mountOptions:
26+
- discard

‎scripts/k8s-storage/volumesnapshotclass-nvmeof.yaml.in‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
apiVersion: snapshot.storage.k8s.io/v1
3+
kind: VolumeSnapshotClass
4+
metadata:
5+
name: k8s-storage-e2e-nvmeof
6+
driver: nvmeof.csi.ceph.com
7+
parameters:
8+
clusterID: @@CLUSTER_ID@@
9+
csi.storage.k8s.io/snapshotter-secret-name: rook-csi-nvmeof-provisioner
10+
csi.storage.k8s.io/snapshotter-secret-namespace: rook-ceph
11+
deletionPolicy: Delete

0 commit comments

Comments
 (0)