Skip to content

Commit 0b386fc

Browse files
bogdandobogdando
committed
nova05: separate pre and post ceph OSCP configs
HCI uses edpm post ceph to also patch the OSCP which looks suboptimal. Separate this into a two steps: * make post-ceph nodeset layer handling EDPM resources only (NodeSet, Services, Secrets, ConfigMap) - remove OSCP layer resources * add control-plane-post-ceph layer for OSCP with Glance RBD + Ceph extraMounts and secret config. Instead of re-including lib/control-plane (which required network-values duplication) provide only required values for lib/control-plane/base. * update README.md stages list from 4 to 6 steps, automation step count from 9 to 10 * add note for defferred Ceph config for control-plane.md * add control-plane post-ceph doc for the dedicated Ceph control-plane update stage * remove control-plane update references for dataplane-post-ceph.md, add prerequisite for control-plane-post-ceph * add control-plane-post-ceph stage between pre-ceph deployment and post-ceph nodeset ini automation vars Signed-off-by: Bohdan Dobrelia <bdobreli@redhat.com>
1 parent 4885ba3 commit 0b386fc

14 files changed

Lines changed: 266 additions & 70 deletions

File tree

‎automation/vars/nova05epsilon.yaml‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,23 @@ vas:
105105
type: playbook
106106
source: "../../hooks/playbooks/fix_swift_endpoint.yml"
107107
inventory: "${HOME}/ci-framework-data/artifacts/ceph_inventory.yml"
108-
109-
- name: edpm-nodeset-post-ceph
110-
path: examples/dt/nova/nova05epsilon
108+
- name: control-plane-post-ceph
109+
path: examples/dt/nova/nova05epsilon/control-plane-post-ceph
111110
wait_conditions:
112111
- >-
113112
oc -n openstack wait osctlplane controlplane
114113
--for condition=Ready
115114
--timeout=20m
115+
values:
116+
- name: service-values
117+
src_file: service-values.yaml
118+
- name: edpm-nodeset-values-post-ceph
119+
src_file: values.yaml
120+
build_output: control-plane-post-ceph.yaml
121+
122+
- name: edpm-nodeset-post-ceph
123+
path: examples/dt/nova/nova05epsilon
124+
wait_conditions:
116125
- >-
117126
oc -n openstack wait osdpns
118127
gpu-computes-edpm

‎dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
apiVersion: v1
3+
data:
4+
ceph.client.openstack.keyring: _ignored_
5+
ceph.conf: _ignored_
6+
kind: Secret
7+
metadata:
8+
name: ceph-conf-files
9+
namespace: openstack
10+
type: Opaque

‎dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1alpha1
3+
kind: Component
4+
5+
transformers:
6+
- |-
7+
apiVersion: builtin
8+
kind: NamespaceTransformer
9+
metadata:
10+
name: _ignored_
11+
namespace: openstack
12+
setRoleBindingSubjects: none
13+
unsetOnly: true
14+
fieldSpecs:
15+
- path: metadata/name
16+
kind: Namespace
17+
create: true
18+
19+
components:
20+
- ../../../../lib/control-plane/base
21+
22+
resources:
23+
- ceph_secret.yaml
24+
25+
replacements:
26+
# Storage class (overrides _replaced_ from lib/control-plane/base)
27+
- source:
28+
kind: ConfigMap
29+
name: service-values
30+
fieldPath: data.storageClass
31+
targets:
32+
- select:
33+
kind: OpenStackControlPlane
34+
fieldPaths:
35+
- spec.storageClass
36+
- spec.glance.template.storage.storageClass
37+
- spec.telemetry.template.metricStorage.monitoringStack.storage.persistent.pvcStorageClass
38+
- spec.designate.template.designateBackendbind9.storageClass
39+
options:
40+
create: true
41+
# OVN bridge mapping (overrides _replaced_ from lib/control-plane/base)
42+
- source:
43+
kind: ConfigMap
44+
name: service-values
45+
fieldPath: data.bridgeName
46+
targets:
47+
- select:
48+
kind: OpenStackControlPlane
49+
fieldPaths:
50+
- spec.ovn.template.ovnController.nicMappings.datacentre
51+
options:
52+
create: true
53+
# Ceph config files -> Secret
54+
- source:
55+
kind: ConfigMap
56+
name: edpm-nodeset-values-post-ceph
57+
fieldPath: data.ceph_conf
58+
targets:
59+
- select:
60+
kind: Secret
61+
name: ceph-conf-files
62+
fieldPaths:
63+
- data
64+
options:
65+
create: true
66+
# Glance RBD backend
67+
- source:
68+
kind: ConfigMap
69+
name: service-values
70+
fieldPath: data.glance.customServiceConfig
71+
targets:
72+
- select:
73+
kind: OpenStackControlPlane
74+
fieldPaths:
75+
- spec.glance.template.customServiceConfig
76+
options:
77+
create: true
78+
- source:
79+
kind: ConfigMap
80+
name: service-values
81+
fieldPath: data.glance.default.replicas
82+
targets:
83+
- select:
84+
kind: OpenStackControlPlane
85+
fieldPaths:
86+
- spec.glance.template.glanceAPIs.default.replicas
87+
options:
88+
create: true
89+
- source:
90+
kind: ConfigMap
91+
name: service-values
92+
fieldPath: data.extraMounts
93+
targets:
94+
- select:
95+
kind: OpenStackControlPlane
96+
fieldPaths:
97+
- spec.extraMounts
98+
options:
99+
create: true

‎dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ transformers:
1818
create: true
1919
2020
components:
21-
- ../../control-plane/
2221
- ../../../../../lib/dataplane/nodeset
2322

2423
resources:

‎dt/nova/nova05epsilon/kustomization.yaml‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,6 @@ replacements:
4343
- spec.ovn.template.ovnController.nicMappings
4444
options:
4545
create: true
46-
- source:
47-
kind: ConfigMap
48-
name: service-values
49-
fieldPath: data.glance.customServiceConfig
50-
targets:
51-
- select:
52-
kind: OpenStackControlPlane
53-
fieldPaths:
54-
- spec.glance.template.customServiceConfig
55-
options:
56-
create: true
5746
- source:
5847
kind: ConfigMap
5948
name: service-values
@@ -131,14 +120,3 @@ replacements:
131120
- spec.nova.template.schedulerServiceTemplate.customServiceConfig
132121
options:
133122
create: true
134-
- source:
135-
kind: ConfigMap
136-
name: service-values
137-
fieldPath: data.extraMounts
138-
targets:
139-
- select:
140-
kind: OpenStackControlPlane
141-
fieldPaths:
142-
- spec.extraMounts
143-
options:
144-
create: true

‎examples/dt/nova/nova05epsilon/README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ This is a collection of CR templates that represent a Red Hat OpenStack Services
2828

2929
5. Between stages 3 and 4, _it is assumed that the user installs Ceph on the compute nodes._ OpenStack K8S CRDs do not provide a way to install Ceph via any sort of combination of CRs.
3030

31-
6. For CI automation, this DT uses `automation/vars/nova05epsilon.yaml` which maps the manual stages above to 9 granular automation steps (NNCP, networking, control-plane, DNS, baremetalhosts, pre-ceph nodeset, pre-ceph deployment, post-ceph nodeset, post-ceph deployment).
31+
6. For CI automation, this DT uses `automation/vars/nova05epsilon.yaml` which maps the manual stages above to 10 granular automation steps (NNCP, networking, control-plane, DNS, baremetalhosts, pre-ceph nodeset, pre-ceph deployment, control-plane-post-ceph, post-ceph nodeset, post-ceph deployment).
3232

3333
7. On SNO with a single EDPM compute (single-host CephHCI), the Ceph ingress service (haproxy/keepalived) is not deployed. The post-ceph hook detects this and corrects the Keystone Swift endpoint. Pods that need to reach RGW (e.g. Tempest) must attach the `storage` and `ctlplane` NetworkAttachmentDefinitions.
3434

@@ -59,7 +59,9 @@ All stages must be executed in the order listed below. Everything is required un
5959
1. [Install the OpenStack K8S operators and their dependencies](../../../common/)
6060
2. [Configuring networking and deploy the OpenStack control plane](control-plane.md)
6161
3. [Configure and deploy the initial data plane to prepare for Ceph installation](dataplane-pre-ceph.md)
62-
4. [Update the control plane and finish deploying the data plane after Ceph has been installed](dataplane-post-ceph.md)
62+
4. Install Ceph on the compute nodes (without changing OpenStack CP CR)
63+
5. [Update the control plane with Ceph backend configuration](control-plane-post-ceph.md)
64+
6. [Finish deploying the data plane after Ceph has been installed](dataplane-post-ceph.md)
6365

6466
## Extending to a Full DCN Deployment
6567

‎examples/dt/nova/nova05epsilon/control-plane-post-ceph.md‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Update the control plane with Ceph backend configuration
2+
3+
## Assumptions
4+
5+
- The [pre-ceph data plane](dataplane-pre-ceph.md) has been deployed
6+
- Ceph has been installed on the compute nodes
7+
- The `ceph-conf-files` secret will be created during this step
8+
9+
## Initialize
10+
11+
Switch to the "openstack" namespace
12+
13+
```shell
14+
oc project openstack
15+
```
16+
17+
Change to the control-plane-post-ceph directory
18+
19+
```shell
20+
cd architecture/examples/dt/nova/nova05epsilon/control-plane-post-ceph
21+
```
22+
23+
Edit the [service-values.yaml](control-plane-post-ceph/service-values.yaml)
24+
file if you need to adjust the Glance RBD backend or Ceph extraMounts
25+
configuration.
26+
27+
```shell
28+
vi service-values.yaml
29+
```
30+
31+
## Update the control plane
32+
33+
Generate the control-plane-post-ceph CRs:
34+
35+
```shell
36+
kustomize build > control-plane-post-ceph.yaml
37+
```
38+
39+
Apply the CRs:
40+
41+
```shell
42+
oc apply -f control-plane-post-ceph.yaml
43+
```
44+
45+
Wait for the control plane to be ready:
46+
47+
```shell
48+
oc wait osctlplane controlplane --for condition=Ready --timeout=1200s
49+
```

‎examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
apiVersion: kustomize.config.k8s.io/v1beta1
3+
kind: Kustomization
4+
5+
components:
6+
- ../../../../../dt/nova/nova05epsilon/control-plane-post-ceph
7+
8+
resources:
9+
- values.yaml
10+
- service-values.yaml

‎examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# local-config: referenced, but not emitted by kustomize
2+
#
3+
# Post-Ceph control plane update: adds Glance RBD backend and
4+
# Ceph extraMounts to the OSCP after the ceph-conf-files secret exists.
5+
---
6+
apiVersion: v1
7+
kind: ConfigMap
8+
metadata:
9+
name: service-values
10+
annotations:
11+
config.kubernetes.io/local-config: "true"
12+
data:
13+
preserveJobs: false
14+
notificationsBus:
15+
cluster: rabbitmq
16+
tls:
17+
caBundleSecretName: ""
18+
storageClass: lvms-local-storage
19+
bridgeName: ospbr
20+
21+
glance:
22+
default:
23+
replicas: 1
24+
customServiceConfig: |
25+
[DEFAULT]
26+
enabled_backends = default_backend:rbd
27+
[glance_store]
28+
default_backend = default_backend
29+
[default_backend]
30+
rbd_store_ceph_conf = /etc/ceph/ceph.conf
31+
store_description = "Ceph RBD backend"
32+
rbd_store_pool = images
33+
rbd_store_user = openstack
34+
rbd_thin_provisioning = True
35+
extraMounts:
36+
- name: v1
37+
region: r1
38+
extraVol:
39+
- propagation:
40+
- GlanceAPI
41+
extraVolType: Ceph
42+
volumes:
43+
- name: ceph
44+
secret:
45+
secretName: ceph-conf-files
46+
mounts:
47+
- name: ceph
48+
mountPath: /etc/ceph
49+
readOnly: true

‎examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# local-config: referenced, but not emitted by kustomize
2+
#
3+
# Ceph config for the control-plane-post-ceph stage.
4+
# In CI, ci_gen_kustomize_values populates ceph_conf from /tmp/ files
5+
# written by the ceph.yml hook (via the edpm-nodeset-values-post-ceph
6+
# Jinja2 template). For manual use, replace CHANGEME values with
7+
# base64-encoded Ceph keyring and conf.
8+
---
9+
apiVersion: v1
10+
kind: ConfigMap
11+
metadata:
12+
name: edpm-nodeset-values-post-ceph
13+
annotations:
14+
config.kubernetes.io/local-config: "true"
15+
data:
16+
ceph_conf:
17+
ceph.client.openstack.keyring: CHANGEME_CEPH_KEYRING
18+
ceph.conf: CHANGEME_CEPH_CONF

0 commit comments

Comments
 (0)