From c8856ffefdd779d8f45fc73cdb6b0fa03e1b40c5 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Wed, 3 Jun 2026 16:48:00 +0200 Subject: [PATCH 1/5] nova05: network config fixes 1. Add cross-site routes/GWs and annotations For all nova05epsilon's NADs, NetConfig subnets, host-level NNCP routes add missing cross-site DCN routes, gateways, and storagemgmt.endpoint_annotation. Previsourely those were tested in ci-framework jobs overlays but the base DT example was missing those CHANGEME templates. 2. Add external provider network for tests accessing VMs Add external NAD definition with VLAN to DT nncp/values.yaml. Update scenario-vars.yaml with external NAD for Tempest access to FIP. Signed-off-by: Bohdan Dobrelia --- .../networking/external-nad.yaml | 8 + .../networking/kustomization.yaml | 13 ++ .../networking/nncp/kustomization.yaml | 167 +++++++++--------- .../control-plane/networking/nncp/values.yaml | 115 +++++++++++- 4 files changed, 213 insertions(+), 90 deletions(-) create mode 100644 dt/nova/nova05epsilon/networking/external-nad.yaml diff --git a/dt/nova/nova05epsilon/networking/external-nad.yaml b/dt/nova/nova05epsilon/networking/external-nad.yaml new file mode 100644 index 000000000..65773c9d1 --- /dev/null +++ b/dt/nova/nova05epsilon/networking/external-nad.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: external + labels: + osp/net: external + osp/net-attach-def-type: standard diff --git a/dt/nova/nova05epsilon/networking/kustomization.yaml b/dt/nova/nova05epsilon/networking/kustomization.yaml index 66eadf45b..0d6ec1126 100644 --- a/dt/nova/nova05epsilon/networking/kustomization.yaml +++ b/dt/nova/nova05epsilon/networking/kustomization.yaml @@ -25,6 +25,7 @@ components: resources: - storagemgmt-nad.yaml - storagemgmt-metallb.yaml + - external-nad.yaml # Add storagemgmt network template, as it is needed for CephHCI patches: @@ -122,3 +123,15 @@ replacements: name: ctlplane fieldPaths: - spec.interfaces.0 + + # External NAD for provider network FIP access + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.net-attach-def + targets: + - select: + kind: NetworkAttachmentDefinition + name: external + fieldPaths: + - spec.config diff --git a/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml b/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml index 50cfe56e4..e7dbb1331 100644 --- a/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml +++ b/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml @@ -40,93 +40,92 @@ patches: kind: NodeNetworkConfigurationPolicy metadata: name: node-2 - # SNO: remove worker scheduling label from nodeSelector + # SNO: remove worker scheduling label from nodeSelector. + # Use labelSelector because lib/nncp renames the resource before + # this component's patches run. - target: kind: NodeNetworkConfigurationPolicy - name: node-0 + labelSelector: "osp/nncm-config-type=standard" patch: |- - op: remove path: /spec/nodeSelector/node-role.kubernetes.io~1worker - # Optional: add storagemgmt VLAN interface to the SNO node. - # Uncomment this patch AND node_0.storagemgmt_ip in values.yaml. - # Not required for CephHCI (Ceph uses storage network for OSD traffic). - # - target: - # kind: NodeNetworkConfigurationPolicy - # name: node-0 - # patch: |- - # - op: add - # path: /spec/desiredState/interfaces/- - # value: - # description: storagemgmt vlan host interface - # name: storagemgmt - # state: up - # type: vlan - # mtu: _replaced_ - # ipv4: - # address: - # - ip: _replaced_ - # prefix-length: _replaced_ - # dhcp: false - # enabled: true - # ipv6: - # enabled: false - # vlan: - # base-iface: _replaced_ - # id: _replaced_ -# Uncomment these replacements together with the storagemgmt patch above -# replacements: -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.node_0.storagemgmt_ip -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# name: node-0 -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.ip -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.base_iface -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].vlan.base-iface -# options: -# create: true -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.vlan -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].vlan.id -# options: -# create: true -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.mtu -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].mtu -# options: -# create: true -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.prefix-length -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.prefix-length -# options: -# create: true + # Add external VLAN interface on ospbr for OVN provider network access. + - target: + kind: NodeNetworkConfigurationPolicy + labelSelector: "osp/nncm-config-type=standard" + patch: |- + - op: add + path: /spec/desiredState/interfaces/- + value: + description: external vlan interface on ospbr + name: external + state: up + type: vlan + mtu: _replaced_ + ipv4: + address: + - ip: _replaced_ + prefix-length: _replaced_ + dhcp: false + enabled: true + ipv6: + enabled: false + vlan: + base-iface: _replaced_ + id: _replaced_ + +replacements: + - source: + kind: ConfigMap + name: network-values + fieldPath: data.node_0.external_ip + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].ipv4.address.0.ip + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.base_iface + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].vlan.base-iface + options: + create: true + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.vlan + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].vlan.id + options: + create: true + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.mtu + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].mtu + options: + create: true + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.prefix-length + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].ipv4.address.0.prefix-length + options: + create: true diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml index e7a8caba2..c5a007f45 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml @@ -30,7 +30,11 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_CTLPLANE_CIDR", "range_start": "CHANGEME_SNO_CTLPLANE_NAD_START", - "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END" + "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_CTLPLANE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" }, + { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" } + ] } }, { "type": "tuning", @@ -50,6 +54,13 @@ data: gateway: CHANGEME_SNO_CTLPLANE_GW name: subnet1 vlan: CHANGEME_SNO_CTLPLANE_VLAN + routes: + - destination: CHANGEME_EDPM_CTLPLANE_CIDR + nexthop: CHANGEME_SNO_CTLPLANE_GW + # Pods with only ctlplane attached (e.g. tempest) need this + # to reach Ceph RGW on the remote storage network. + - destination: CHANGEME_EDPM_STORAGE_CIDR + nexthop: CHANGEME_SNO_CTLPLANE_GW # CHANGEME: site4 EDPM computes -- replace CIDRs/gateway/VLAN - allocationRanges: - end: CHANGEME_EDPM_CTLPLANE_END @@ -58,6 +69,9 @@ data: gateway: CHANGEME_EDPM_CTLPLANE_GW name: subnet2 vlan: CHANGEME_EDPM_CTLPLANE_VLAN + routes: + - destination: CHANGEME_SNO_CTLPLANE_CIDR + nexthop: CHANGEME_EDPM_CTLPLANE_GW datacentre: net-attach-def: | { @@ -84,8 +98,31 @@ data: values: - CHANGEME_SNO_DNS_SERVER external: + base_iface: ospbr dnsDomain: external.example.com + iface: external mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "external", + "plugins": [{ + "type": "macvlan", + "master": "external", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_EXTERNAL_CIDR", + "range_start": "CHANGEME_SNO_EXTERNAL_NAD_START", + "range_end": "CHANGEME_SNO_EXTERNAL_NAD_END" + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } prefix-length: 24 subnets: - allocationRanges: @@ -126,7 +163,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_INTAPI_CIDR", "range_start": "CHANGEME_SNO_INTAPI_NAD_START", - "range_end": "CHANGEME_SNO_INTAPI_NAD_END" + "range_end": "CHANGEME_SNO_INTAPI_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_INTAPI_CIDR", "gw": "CHANGEME_SNO_INTAPI_GW" } + ] } }, { "type": "tuning", @@ -141,20 +181,29 @@ data: - end: CHANGEME_SNO_INTAPI_END start: CHANGEME_SNO_INTAPI_START cidr: CHANGEME_SNO_INTAPI_CIDR + gateway: CHANGEME_SNO_INTAPI_GW name: subnet1 vlan: CHANGEME_SNO_INTAPI_VLAN + routes: + - destination: CHANGEME_EDPM_INTAPI_CIDR + nexthop: CHANGEME_SNO_INTAPI_GW # CHANGEME: site4 internalapi subnet - allocationRanges: - end: CHANGEME_EDPM_INTAPI_END start: CHANGEME_EDPM_INTAPI_START cidr: CHANGEME_EDPM_INTAPI_CIDR + gateway: CHANGEME_EDPM_INTAPI_GW name: subnet2 vlan: CHANGEME_EDPM_INTAPI_VLAN + routes: + - destination: CHANGEME_SNO_INTAPI_CIDR + nexthop: CHANGEME_EDPM_INTAPI_GW vlan: CHANGEME_SNO_INTAPI_VLAN lbServiceType: LoadBalancer # SNO: single OCP node only node_0: ctlplane_ip: CHANGEME_SNO_CTLPLANE_IP + external_ip: CHANGEME_SNO_EXTERNAL_IP internalapi_ip: CHANGEME_SNO_INTAPI_IP name: CHANGEME_SNO_NODE_NAME storage_ip: CHANGEME_SNO_STORAGE_IP @@ -183,7 +232,24 @@ data: metallb.universe.tf/address-pool: internalapi metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_RABBITMQ_CELL1_IP routes: - config: [] + config: + # CHANGEME: host-level routes for cross-site reachability (EDPM subnet2) + - destination: CHANGEME_EDPM_CTLPLANE_CIDR + next-hop-address: CHANGEME_SNO_CTLPLANE_GW + next-hop-interface: ospbr + table-id: 254 + - destination: CHANGEME_EDPM_INTAPI_CIDR + next-hop-address: CHANGEME_SNO_INTAPI_GW + next-hop-interface: internalapi + table-id: 254 + - destination: CHANGEME_EDPM_STORAGE_CIDR + next-hop-address: CHANGEME_SNO_STORAGE_GW + next-hop-interface: storage + table-id: 254 + - destination: CHANGEME_EDPM_TENANT_CIDR + next-hop-address: CHANGEME_SNO_TENANT_GW + next-hop-interface: tenant + table-id: 254 storage: base_iface: CHANGEME_SNO_IFACE dnsDomain: storage.example.com @@ -206,7 +272,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_STORAGE_CIDR", "range_start": "CHANGEME_SNO_STORAGE_NAD_START", - "range_end": "CHANGEME_SNO_STORAGE_NAD_END" + "range_end": "CHANGEME_SNO_STORAGE_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_STORAGE_GW" } + ] } }, { "type": "tuning", @@ -221,20 +290,32 @@ data: - end: CHANGEME_SNO_STORAGE_END start: CHANGEME_SNO_STORAGE_START cidr: CHANGEME_SNO_STORAGE_CIDR + gateway: CHANGEME_SNO_STORAGE_GW name: subnet1 vlan: CHANGEME_SNO_STORAGE_VLAN + routes: + - destination: CHANGEME_EDPM_STORAGE_CIDR + nexthop: CHANGEME_SNO_STORAGE_GW # CHANGEME: site4 storage subnet - allocationRanges: - end: CHANGEME_EDPM_STORAGE_END start: CHANGEME_EDPM_STORAGE_START cidr: CHANGEME_EDPM_STORAGE_CIDR + gateway: CHANGEME_EDPM_STORAGE_GW name: subnet2 vlan: CHANGEME_EDPM_STORAGE_VLAN + routes: + - destination: CHANGEME_SNO_STORAGE_CIDR + nexthop: CHANGEME_EDPM_STORAGE_GW vlan: CHANGEME_SNO_STORAGE_VLAN storageClass: lvms-local-storage storagemgmt: base_iface: CHANGEME_SNO_IFACE dnsDomain: storagemgmt.example.com + endpoint_annotations: + metallb.universe.tf/address-pool: storagemgmt + metallb.universe.tf/allow-shared-ip: storagemgmt + metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_STGMGMT_LB_IP iface: storagemgmt lb_addresses: - CHANGEME_SNO_STGMGMT_LB_START-CHANGEME_SNO_STGMGMT_LB_END @@ -250,7 +331,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_STGMGMT_CIDR", "range_start": "CHANGEME_SNO_STGMGMT_NAD_START", - "range_end": "CHANGEME_SNO_STGMGMT_NAD_END" + "range_end": "CHANGEME_SNO_STGMGMT_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_STGMGMT_CIDR", "gw": "CHANGEME_SNO_STGMGMT_GW" } + ] } }, { "type": "tuning", @@ -265,15 +349,23 @@ data: - end: CHANGEME_SNO_STGMGMT_END start: CHANGEME_SNO_STGMGMT_START cidr: CHANGEME_SNO_STGMGMT_CIDR + gateway: CHANGEME_SNO_STGMGMT_GW name: subnet1 vlan: CHANGEME_SNO_STGMGMT_VLAN + routes: + - destination: CHANGEME_EDPM_STGMGMT_CIDR + nexthop: CHANGEME_SNO_STGMGMT_GW # CHANGEME: site4 storagemgmt subnet - allocationRanges: - end: CHANGEME_EDPM_STGMGMT_END start: CHANGEME_EDPM_STGMGMT_START cidr: CHANGEME_EDPM_STGMGMT_CIDR + gateway: CHANGEME_EDPM_STGMGMT_GW name: subnet2 vlan: CHANGEME_EDPM_STGMGMT_VLAN + routes: + - destination: CHANGEME_SNO_STGMGMT_CIDR + nexthop: CHANGEME_EDPM_STGMGMT_GW vlan: CHANGEME_SNO_STGMGMT_VLAN tenant: base_iface: CHANGEME_SNO_IFACE @@ -297,7 +389,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_TENANT_CIDR", "range_start": "CHANGEME_SNO_TENANT_NAD_START", - "range_end": "CHANGEME_SNO_TENANT_NAD_END" + "range_end": "CHANGEME_SNO_TENANT_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_TENANT_CIDR", "gw": "CHANGEME_SNO_TENANT_GW" } + ] } }, { "type": "tuning", @@ -312,15 +407,23 @@ data: - end: CHANGEME_SNO_TENANT_END start: CHANGEME_SNO_TENANT_START cidr: CHANGEME_SNO_TENANT_CIDR + gateway: CHANGEME_SNO_TENANT_GW name: subnet1 vlan: CHANGEME_SNO_TENANT_VLAN + routes: + - destination: CHANGEME_EDPM_TENANT_CIDR + nexthop: CHANGEME_SNO_TENANT_GW # CHANGEME: site4 tenant subnet - allocationRanges: - end: CHANGEME_EDPM_TENANT_END start: CHANGEME_EDPM_TENANT_START cidr: CHANGEME_EDPM_TENANT_CIDR + gateway: CHANGEME_EDPM_TENANT_GW name: subnet2 vlan: CHANGEME_EDPM_TENANT_VLAN + routes: + - destination: CHANGEME_SNO_TENANT_CIDR + nexthop: CHANGEME_EDPM_TENANT_GW vlan: CHANGEME_SNO_TENANT_VLAN kind: ConfigMap From f10368a57f55b214f7f45419d61f43bc729b0dde Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Fri, 5 Jun 2026 16:26:43 +0200 Subject: [PATCH 2/5] nova05: add automation mocks for CI Signed-off-by: Bohdan Dobrelia --- automation/mocks/nova05epsilon.yaml | 42 ++ automation/net-env/nova05epsilon.yaml | 477 ++++++++++++++++++ .../control-plane/networking/nncp/values.yaml | 4 - zuul.d/validations.yaml | 3 + 4 files changed, 522 insertions(+), 4 deletions(-) create mode 100644 automation/mocks/nova05epsilon.yaml create mode 100644 automation/net-env/nova05epsilon.yaml diff --git a/automation/mocks/nova05epsilon.yaml b/automation/mocks/nova05epsilon.yaml new file mode 100644 index 000000000..654294e64 --- /dev/null +++ b/automation/mocks/nova05epsilon.yaml @@ -0,0 +1,42 @@ +--- +cifmw_install_ca_url: http://example.com/example.pem +cifmw_repo_setup_rhos_release_rpm: http://example.com/rhos-release.rpm +cifmw_repo_setup_rhos_release_args: example-args +cifmw_ci_gen_kustomize_values_remove_keys_expressions: + - ^node(_[0-9]+)?$ +cifmw_baremetal_hosts: + compute-0: + connection: idrac-virtualmedia://foo.example.com/redfish/v1/Systems/System.Embedded.1 + boot_mode: UEFI + status: running + nics: + - mac: "aa:bb:cc:dd:ee:ff" + network: provision + root_device_hint: /dev/disk/by-path/pci-0000:65:00.0-scsi-0:1:234:5 + root_device_hint_field: deviceName + credentials_file: "/home/zuul/idrac_access.yaml" + nmstate: + interfaces: + - name: nic1 + type: ethernet + state: up + ipv4: + enabled: true + dhcp: true + - name: nic2 + type: ethernet + state: up + ipv4: + enabled: true + address: + - ip: 10.0.0.10 + prefix-length: 24 + dns-resolver: + config: + server: + - 10.0.0.1 + routes: + config: + - destination: 0.0.0.0/0 + next-hop-address: 10.0.0.1 + next-hop-interface: nic1 diff --git a/automation/net-env/nova05epsilon.yaml b/automation/net-env/nova05epsilon.yaml new file mode 100644 index 000000000..89a9ebc1d --- /dev/null +++ b/automation/net-env/nova05epsilon.yaml @@ -0,0 +1,477 @@ +--- +instances: + compute-0: + name: compute-0 + networks: + ctlplane: + interface_name: eth1 + ip_v4: 192.168.122.100 + is_trunk_parent: true + mac_addr: 52:54:03:b1:78:6c + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: eth1.20 + ip_v4: 172.17.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:1e:58:da + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + storage: + interface_name: eth1.21 + ip_v4: 172.18.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:7e:be:ad + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + storagemgmt: + interface_name: eth1.23 + ip_v4: 172.20.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:aa:bb:cc + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storagemgmt + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 23 + tenant: + interface_name: eth1.22 + ip_v4: 172.19.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:33:ee:37 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + external: + interface_name: eth1.44 + ip_v4: 10.0.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:dd:ee:ff + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: external + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 44 + ocp-master-0: + hostname: master-0 + name: ocp-master-0 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.10 + is_trunk_parent: true + mac_addr: 52:54:00:15:07:f4 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:68:4b:ce + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:05:23:2c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + storagemgmt: + interface_name: enp6s0.23 + ip_v4: 172.20.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:11:22:33 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storagemgmt + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 23 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:06:80:f0 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + external: + interface_name: enp6s0.44 + ip_v4: 10.0.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:44:55:66 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: external + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 44 + # unused but required by lib components + ocp-master-1: + hostname: master-1 + name: ocp-master-1 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.11 + is_trunk_parent: true + mac_addr: 52:54:01:43:50:83 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:71:78:e6 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.11 + mac_addr: 52:54:01:51:68:1e + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:76:3d:ba + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:7a:5d:1d + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-2: + hostname: master-2 + name: ocp-master-2 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.12 + is_trunk_parent: true + mac_addr: 52:54:02:fd:b8:5a + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:66:a0:7c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.12 + mac_addr: 52:54:02:50:e9:87 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:09:45:3b + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:22:70:9f + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + +networks: + ctlplane: + dns_v4: + - 192.168.122.1 + dns_v6: [] + gw_v4: 192.168.122.1 + mtu: 1500 + network_name: ctlplane + network_v4: 192.168.122.0/24 + search_domain: ctlplane.example.com + tools: + metallb: + ipv4_ranges: + - end: 192.168.122.90 + end_host: 90 + length: 11 + start: 192.168.122.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 192.168.122.70 + end_host: 70 + length: 41 + start: 192.168.122.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 192.168.122.120 + end_host: 120 + length: 21 + start: 192.168.122.100 + start_host: 100 + - end: 192.168.122.170 + end_host: 170 + length: 21 + start: 192.168.122.150 + start_host: 150 + ipv6_ranges: [] + external: + dns_v4: [] + dns_v6: [] + mtu: 1500 + network_name: external + network_v4: 10.0.0.0/24 + search_domain: external.example.com + tools: + multus: + ipv4_ranges: + - end: 10.0.0.70 + end_host: 70 + length: 41 + start: 10.0.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 10.0.0.250 + end_host: 250 + length: 151 + start: 10.0.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 44 + internalapi: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: internalapi + network_v4: 172.17.0.0/24 + search_domain: internalapi.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.17.0.90 + end_host: 90 + length: 11 + start: 172.17.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.17.0.70 + end_host: 70 + length: 41 + start: 172.17.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.17.0.250 + end_host: 250 + length: 151 + start: 172.17.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 20 + storage: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: storage + network_v4: 172.18.0.0/24 + search_domain: storage.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.18.0.90 + end_host: 90 + length: 11 + start: 172.18.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.18.0.70 + end_host: 70 + length: 41 + start: 172.18.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.18.0.250 + end_host: 250 + length: 151 + start: 172.18.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 21 + storagemgmt: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: storagemgmt + network_v4: 172.20.0.0/24 + search_domain: storagemgmt.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.20.0.90 + end_host: 90 + length: 11 + start: 172.20.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.20.0.70 + end_host: 70 + length: 41 + start: 172.20.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.20.0.250 + end_host: 250 + length: 151 + start: 172.20.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 23 + tenant: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: tenant + network_v4: 172.19.0.0/24 + search_domain: tenant.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.19.0.90 + end_host: 90 + length: 11 + start: 172.19.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.19.0.70 + end_host: 70 + length: 41 + start: 172.19.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.19.0.250 + end_host: 250 + length: 151 + start: 172.19.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 22 +routers: {} diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml index c5a007f45..66b1a66de 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml @@ -312,10 +312,6 @@ data: storagemgmt: base_iface: CHANGEME_SNO_IFACE dnsDomain: storagemgmt.example.com - endpoint_annotations: - metallb.universe.tf/address-pool: storagemgmt - metallb.universe.tf/allow-shared-ip: storagemgmt - metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_STGMGMT_LB_IP iface: storagemgmt lb_addresses: - CHANGEME_SNO_STGMGMT_LB_START-CHANGEME_SNO_STGMGMT_LB_END diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml index 8db00ed37..1e4826c1e 100644 --- a/zuul.d/validations.yaml +++ b/zuul.d/validations.yaml @@ -307,6 +307,8 @@ cifmw_architecture_scenario: nova04delta-adoption - job: files: + - automation/mocks/nova05epsilon.yaml + - automation/net-env/nova05epsilon.yaml - examples/dt/nova/nova05epsilon - examples/dt/nova/nova05epsilon/control-plane - examples/dt/nova/nova05epsilon/control-plane/networking @@ -321,6 +323,7 @@ parent: rhoso-architecture-base-job vars: cifmw_architecture_scenario: nova05epsilon + cifmw_networking_env_def_file: automation/net-env/nova05epsilon.yaml - job: files: - examples/va/nvidia-mdev/control-plane From c1a6e5c5c9a29de13057549bcc9601200c36c6b8 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Tue, 9 Jun 2026 18:09:19 +0200 Subject: [PATCH 3/5] nova05: fix telemetry spec and kustomize chain for SNO The DT-level kustomization was missing service-values replacements (telemetry, nova, glance, ovn, extraMounts), so the examples kustomize build produced base defaults instead of DT-specific config. This caused metricStorage.enabled=false and missing alertmanagerConfig on the deployed OSCP. Additionally, pvcStorageClass was hardcoded to "local-storage" but SNO uses LVMS ("lvms-local-storage"), and alertmanager defaulted to 2 replicas which cannot schedule on a single node due to required pod anti-affinity. Align the kustomization layers with nova04delta: - dt/nova/nova05epsilon/kustomization.yaml: add service-values replacements matching nova04delta pattern - Remove deplicated replacements for dt/nova/nova05epsilon/control-plane/kustomization.yaml - examples/.../control-plane/service-values.yaml: add full DT-specific data (telemetry, nova, glance, ovn, extraMounts) - Set pvcStorageClass to lvms-local-storage and alertmanagerConfig replicas to 1 for SNO compatibility - update control-plane.md Generated-by: claude-4.6-opus-high Signed-off-by: Bohdan Dobrelia --- .../control-plane/kustomization.yaml | 130 +----------------- dt/nova/nova05epsilon/kustomization.yaml | 123 +++++++++++++++++ .../dt/nova/nova05epsilon/control-plane.md | 10 +- .../control-plane/service-values.yaml | 88 ++++++++++++ .../dt/nova/nova05epsilon/service-values.yaml | 4 +- 5 files changed, 220 insertions(+), 135 deletions(-) diff --git a/dt/nova/nova05epsilon/control-plane/kustomization.yaml b/dt/nova/nova05epsilon/control-plane/kustomization.yaml index 7a69f6f50..b74de7e00 100644 --- a/dt/nova/nova05epsilon/control-plane/kustomization.yaml +++ b/dt/nova/nova05epsilon/control-plane/kustomization.yaml @@ -18,132 +18,4 @@ transformers: create: true components: - - ../../../../lib/control-plane/base - - ../../../../lib/control-plane/service-endpoints - - ../../../../lib/control-plane/dns - - ../../../../lib/control-plane/storage - - ../../../../lib/control-plane/ovn-bridge - - ../../../../lib/control-plane/job-settings - -replacements: - - source: - kind: ConfigMap - name: service-values - fieldPath: data.neutron.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.neutron.template.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.ovn.ovnController.nicMappings - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.ovn.template.ovnController.nicMappings - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.glance.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.glance.template.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.glance.default.replicas - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.glance.template.glanceAPIs.default.replicas - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.swift.enabled - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.swift.enabled - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.telemetry - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.telemetry - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.apiServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.apiServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.cell0.conductorServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.cellTemplates.cell0.conductorServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.cell1.conductorServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.cellTemplates.cell1.conductorServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.schedulerServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.extraMounts - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.extraMounts - options: - create: true + - ../../../../lib/control-plane diff --git a/dt/nova/nova05epsilon/kustomization.yaml b/dt/nova/nova05epsilon/kustomization.yaml index 2e88f79fb..2d9f6c22a 100644 --- a/dt/nova/nova05epsilon/kustomization.yaml +++ b/dt/nova/nova05epsilon/kustomization.yaml @@ -19,3 +19,126 @@ transformers: components: - ../../../lib/control-plane + +replacements: + - source: + kind: ConfigMap + name: service-values + fieldPath: data.neutron.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.neutron.template.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.ovn.ovnController.nicMappings + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.ovn.template.ovnController.nicMappings + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.default.replicas + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.glanceAPIs.default.replicas + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.swift.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.swift.enabled + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.telemetry + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.telemetry + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.apiServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.apiServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell0.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell0.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell1.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell1.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.schedulerServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.extraMounts + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.extraMounts + options: + create: true diff --git a/examples/dt/nova/nova05epsilon/control-plane.md b/examples/dt/nova/nova05epsilon/control-plane.md index 3697b0f7f..d731c3f64 100644 --- a/examples/dt/nova/nova05epsilon/control-plane.md +++ b/examples/dt/nova/nova05epsilon/control-plane.md @@ -18,15 +18,15 @@ Change to the nova05epsilon directory cd architecture/examples/dt/nova/nova05epsilon/control-plane ``` -Edit the [networking/nncp/values.yaml](control-plane/networking/nncp/values.yaml) -and [networking/dns/values.yaml](control-plane/networking/dns/values.yaml) -files to suit your environment. Service values (Glance, Nova PCI, telemetry) -are configured in the top-level [service-values.yaml](service-values.yaml) -and applied during the post-ceph stage rebuild. +Edit the [networking/nncp/values.yaml](control-plane/networking/nncp/values.yaml), +[networking/dns/values.yaml](control-plane/networking/dns/values.yaml) and +[service-values.yaml](control-plane/service-values.yaml) files to suit +your environment. ```shell vi networking/nncp/values.yaml vi networking/dns/values.yaml +vi service-values.yaml ``` ## Apply node network configuration diff --git a/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml b/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml index c87917e6b..d531c5d0c 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml @@ -14,3 +14,91 @@ data: cluster: rabbitmq tls: caBundleSecretName: "" + + # --- Below are VA/DT-specific data --- + neutron: + customServiceConfig: | + [ml2] + mechanism_drivers = ovn + ovn: + ovnController: + nicMappings: + datacentre: ospbr + glance: + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:rbd + [glance_store] + default_backend = default_backend + [default_backend] + rbd_store_ceph_conf = /etc/ceph/ceph.conf + store_description = "Ceph RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + default: + replicas: 1 + swift: + enabled: false + telemetry: + enabled: true + template: + ceilometer: + enabled: true + logging: + enabled: false + metricStorage: + dashboardsEnabled: true + enabled: true + dataplaneNetwork: ctlplane + networkAttachments: + - ctlplane + monitoringStack: + alertingEnabled: true + alertmanagerConfig: + replicas: 1 + scrapeInterval: 30s + storage: + persistent: + pvcStorageClass: lvms-local-storage + pvcStorageRequest: 20G + pvcStorageSelector: {} + retention: 24h + strategy: persistent + # Nova GPU PCI passthrough configuration + nova: + apiServiceTemplate: + customServiceConfig: | + [pci] + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + [filter_scheduler] + pci_in_placement = True + cell0: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + cell1: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + extraMounts: + - name: v1 + region: r1 + extraVol: + - propagation: + - GlanceAPI + extraVolType: Ceph + volumes: + - name: ceph + secret: + secretName: ceph-conf-files + mounts: + - name: ceph + mountPath: /etc/ceph + readOnly: true diff --git a/examples/dt/nova/nova05epsilon/service-values.yaml b/examples/dt/nova/nova05epsilon/service-values.yaml index 6033bb80c..572766665 100644 --- a/examples/dt/nova/nova05epsilon/service-values.yaml +++ b/examples/dt/nova/nova05epsilon/service-values.yaml @@ -55,10 +55,12 @@ data: - ctlplane monitoringStack: alertingEnabled: true + alertmanagerConfig: + replicas: 1 scrapeInterval: 30s storage: persistent: - pvcStorageClass: local-storage + pvcStorageClass: lvms-local-storage pvcStorageRequest: 20G pvcStorageSelector: {} retention: 24h From f8605751271bf6b1c99b16c83300a9052e682d80 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Wed, 10 Jun 2026 12:01:02 +0200 Subject: [PATCH 4/5] nova05: separate pre and post ceph OSCP configs HCI uses edpm post ceph to also patch the OSCP which looks suboptimal. Separate this into a two steps: * make post-ceph nodeset layer handling EDPM resources only (NodeSet, Services, Secrets, ConfigMap) - remove OSCP layer resources * add control-plane-post-ceph layer for OSCP with Glance RBD + Ceph extraMounts and secret config. Instead of re-including lib/control-plane (which required network-values duplication) provide only required values for lib/control-plane/base. * update README.md stages list from 4 to 6 steps, automation step count from 9 to 10 * add note for defferred Ceph config for control-plane.md * add control-plane post-ceph doc for the dedicated Ceph control-plane update stage * remove control-plane update references for dataplane-post-ceph.md, add prerequisite for control-plane-post-ceph * add control-plane-post-ceph stage between pre-ceph deployment and post-ceph nodeset ini automation vars Signed-off-by: Bohdan Dobrelia --- automation/vars/nova05epsilon.yaml | 17 +- .../control-plane-post-ceph/ceph_secret.yaml | 10 + .../kustomization.yaml | 160 +++++++ .../edpm-post-ceph/nodeset/kustomization.yaml | 1 - dt/nova/nova05epsilon/kustomization.yaml | 22 - examples/dt/nova/nova05epsilon/README.md | 19 +- .../nova05epsilon/control-plane-post-ceph.md | 79 ++++ .../kustomization.yaml | 11 + .../network-values.yaml | 429 ++++++++++++++++++ .../service-values.yaml | 101 +++++ .../control-plane-post-ceph/values.yaml | 18 + .../dt/nova/nova05epsilon/control-plane.md | 6 +- .../control-plane/service-values.yaml | 30 +- .../nova/nova05epsilon/dataplane-post-ceph.md | 20 +- zuul.d/validations.yaml | 1 + 15 files changed, 854 insertions(+), 70 deletions(-) create mode 100644 dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml create mode 100644 dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml create mode 100644 examples/dt/nova/nova05epsilon/control-plane-post-ceph.md create mode 100644 examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml create mode 100644 examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml create mode 100644 examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml create mode 100644 examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml diff --git a/automation/vars/nova05epsilon.yaml b/automation/vars/nova05epsilon.yaml index c3b507269..3a87f0aab 100644 --- a/automation/vars/nova05epsilon.yaml +++ b/automation/vars/nova05epsilon.yaml @@ -101,14 +101,25 @@ vas: type: playbook source: "../../hooks/playbooks/ceph.yml" inventory: "${HOME}/ci-framework-data/artifacts/ceph_inventory.yml" - - - name: edpm-nodeset-post-ceph - path: examples/dt/nova/nova05epsilon + - name: control-plane-post-ceph + path: examples/dt/nova/nova05epsilon/control-plane-post-ceph wait_conditions: - >- oc -n openstack wait osctlplane controlplane --for condition=Ready --timeout=20m + values: + - name: network-values + src_file: network-values.yaml + - name: service-values + src_file: service-values.yaml + - name: edpm-nodeset-values-post-ceph + src_file: values.yaml + build_output: control-plane-post-ceph.yaml + + - name: edpm-nodeset-post-ceph + path: examples/dt/nova/nova05epsilon + wait_conditions: - >- oc -n openstack wait osdpns gpu-computes-edpm diff --git a/dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml b/dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml new file mode 100644 index 000000000..4d2073239 --- /dev/null +++ b/dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +data: + ceph.client.openstack.keyring: _ignored_ + ceph.conf: _ignored_ +kind: Secret +metadata: + name: ceph-conf-files + namespace: openstack +type: Opaque diff --git a/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml b/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml new file mode 100644 index 000000000..756080b46 --- /dev/null +++ b/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml @@ -0,0 +1,160 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../lib/control-plane/base + - ../../../../lib/control-plane/service-endpoints + - ../../../../lib/control-plane/dns + - ../../../../lib/control-plane/storage + - ../../../../lib/control-plane/ovn-bridge + - ../../../../lib/control-plane/job-settings + - ../../../../lib/control-plane/tls + - ../../../../lib/control-plane/messaging-bus + +resources: + - ceph_secret.yaml + +replacements: + # Ceph config files -> Secret + - source: + kind: ConfigMap + name: edpm-nodeset-values-post-ceph + fieldPath: data.ceph_conf + targets: + - select: + kind: Secret + name: ceph-conf-files + fieldPaths: + - data + options: + create: true + # Glance RBD backend + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.default.replicas + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.glanceAPIs.default.replicas + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.extraMounts + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.extraMounts + options: + create: true + # Swift + - source: + kind: ConfigMap + name: service-values + fieldPath: data.swift.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.swift.enabled + options: + create: true + # Telemetry + - source: + kind: ConfigMap + name: service-values + fieldPath: data.telemetry + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.telemetry + options: + create: true + # Neutron ML2 config + - source: + kind: ConfigMap + name: service-values + fieldPath: data.neutron.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.neutron.template.customServiceConfig + options: + create: true + # Nova PCI passthrough / scheduler config + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.apiServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.apiServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell0.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell0.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell1.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell1.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.schedulerServiceTemplate.customServiceConfig + options: + create: true diff --git a/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml b/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml index 70ab2f823..e14426ab0 100644 --- a/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml +++ b/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml @@ -18,7 +18,6 @@ transformers: create: true components: - - ../../control-plane/ - ../../../../../lib/dataplane/nodeset resources: diff --git a/dt/nova/nova05epsilon/kustomization.yaml b/dt/nova/nova05epsilon/kustomization.yaml index 2d9f6c22a..014f43f6e 100644 --- a/dt/nova/nova05epsilon/kustomization.yaml +++ b/dt/nova/nova05epsilon/kustomization.yaml @@ -43,17 +43,6 @@ replacements: - spec.ovn.template.ovnController.nicMappings options: create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.glance.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.glance.template.customServiceConfig - options: - create: true - source: kind: ConfigMap name: service-values @@ -131,14 +120,3 @@ replacements: - spec.nova.template.schedulerServiceTemplate.customServiceConfig options: create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.extraMounts - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.extraMounts - options: - create: true diff --git a/examples/dt/nova/nova05epsilon/README.md b/examples/dt/nova/nova05epsilon/README.md index 645557a36..371491e71 100644 --- a/examples/dt/nova/nova05epsilon/README.md +++ b/examples/dt/nova/nova05epsilon/README.md @@ -28,7 +28,20 @@ This is a collection of CR templates that represent a Red Hat OpenStack Services 5. Between stages 3 and 4, _it is assumed that the user installs Ceph on the compute nodes._ OpenStack K8S CRDs do not provide a way to install Ceph via any sort of combination of CRs. -6. For CI automation, this DT uses `automation/vars/nova05epsilon.yaml` which maps the manual stages above to 9 granular automation steps (NNCP, networking, control-plane, DNS, baremetalhosts, pre-ceph nodeset, pre-ceph deployment, post-ceph nodeset, post-ceph deployment). +6. In stage 5 the `control-plane-post-ceph` kustomization needs the same network values used in stage 2 to preserve endpoint IPs, service types, and DNS configuration. For manual deployment, copy your environment-customized `control-plane/networking/nncp/values.yaml` into `control-plane-post-ceph/network-values.yaml` and populate `control-plane-post-ceph/values.yaml` with base64-encoded Ceph keyring and config. In CI, `ci_gen_kustomize_values` generates `network-values.yaml` in-place using the common Jinja2 template with environment overlays. See [control-plane-post-ceph.md](control-plane-post-ceph.md) for details. + +7. On SNO with a single EDPM compute (single-host CephHCI), the Ceph ingress service (haproxy/keepalived) is not deployed. The default Swift endpoint (`:8080`) is unreachable because no ingress fronts the RGW daemon. Instead, clients must reach RGW directly on the compute's storage IP at port 8082 (the `rgw_frontend_port` set in the Ceph RGW spec). + For CI automation, set `cifmw_cephadm_rgw_port: 8082` and `cifmw_cephadm_rgw_vip: ` in the scenario vars so that `cifmw_cephadm` creates the Keystone endpoint with the correct address. + For manual deployment, after installing Ceph, update the Swift endpoints in Keystone to point at the RGW daemon directly: + + ```shell + STORAGE_IP= + for ep_id in $(openstack endpoint list --service object-store -f value -c ID); do + openstack endpoint set --url "http://${STORAGE_IP}:8082/swift/v1/AUTH_%(tenant_id)s" "$ep_id" + done + ``` + +8. For CI automation, this DT uses `automation/vars/nova05epsilon.yaml` which maps the manual stages above to 10 granular automation steps (NNCP, networking, control-plane, DNS, baremetalhosts, pre-ceph nodeset, pre-ceph deployment, control-plane-post-ceph, post-ceph nodeset, post-ceph deployment). ## Host Configuration @@ -57,7 +70,9 @@ All stages must be executed in the order listed below. Everything is required un 1. [Install the OpenStack K8S operators and their dependencies](../../../common/) 2. [Configuring networking and deploy the OpenStack control plane](control-plane.md) 3. [Configure and deploy the initial data plane to prepare for Ceph installation](dataplane-pre-ceph.md) -4. [Update the control plane and finish deploying the data plane after Ceph has been installed](dataplane-post-ceph.md) +4. Install Ceph on the compute nodes (without changing OpenStack CP CR) +5. [Update the control plane with Ceph backend configuration](control-plane-post-ceph.md) +6. [Finish deploying the data plane after Ceph has been installed](dataplane-post-ceph.md) ## Extending to a Full DCN Deployment diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph.md b/examples/dt/nova/nova05epsilon/control-plane-post-ceph.md new file mode 100644 index 000000000..7a5743e02 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph.md @@ -0,0 +1,79 @@ +# Update the control plane with Ceph backend configuration + +## Assumptions + +- The [pre-ceph data plane](dataplane-pre-ceph.md) has been deployed +- Ceph has been installed on the compute nodes +- The `ceph-conf-files` secret will be created during this step + +## Initialize + +Switch to the "openstack" namespace + +```shell +oc project openstack +``` + +Change to the control-plane-post-ceph directory + +```shell +cd architecture/examples/dt/nova/nova05epsilon/control-plane-post-ceph +``` + +## Prepare values files + +The kustomization requires three values files: + +**network-values.yaml** — Provides storageClass, bridgeName, endpoint +annotations, and DNS options that the lib components need. This file +must match the `network-values` used in the initial control-plane stage +to preserve all endpoint IPs, service types, and DNS configuration. +In CI, `ci_gen_kustomize_values` generates it from the automation vars +(`src_file: network-values.yaml`). For manual deployment, copy the +environment-customized file from the control-plane networking stage: + +```shell +cp ../control-plane/networking/nncp/values.yaml network-values.yaml +``` + +Edit `network-values.yaml` and replace any remaining `CHANGEME` +placeholders to match your environment (same values used in stage 2). + +**values.yaml** — Ceph configuration for the `ceph-conf-files` secret. +Replace the `CHANGEME` placeholders with base64-encoded Ceph keyring +and config from your Ceph deployment: + +```shell +vi values.yaml +# Set data.ceph_conf."ceph.client.openstack.keyring" to: +# base64 -w0 /etc/ceph/ceph.client.openstack.keyring +# Set data.ceph_conf."ceph.conf" to: +# base64 -w0 /etc/ceph/ceph.conf +``` + +**service-values.yaml** — Edit if you need to adjust the Glance RBD +backend or Ceph extraMounts configuration: + +```shell +vi service-values.yaml +``` + +## Update the control plane + +Generate the control-plane-post-ceph CRs: + +```shell +kustomize build > control-plane-post-ceph.yaml +``` + +Apply the CRs: + +```shell +oc apply -f control-plane-post-ceph.yaml +``` + +Wait for the control plane to be ready: + +```shell +oc wait osctlplane controlplane --for condition=Ready --timeout=1200s +``` diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml new file mode 100644 index 000000000..9d3f4689a --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../dt/nova/nova05epsilon/control-plane-post-ceph + +resources: + - network-values.yaml + - values.yaml + - service-values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml new file mode 100644 index 000000000..66b1a66de --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml @@ -0,0 +1,429 @@ +# Network values for SNO + DCN spine-leaf topology. +# +# Site 2 (SNO control plane): subnet1 +# Site 4 (EDPM computes): subnet2 +# +# Replace all CHANGEME values to match your environment. +# All subnets must share a common routable address space. +--- +apiVersion: v1 +data: + bridgeName: ospbr + ctlplane: + dnsDomain: ctlplane.example.com + endpoint_annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_CTLPLANE_LB_IP + iface: CHANGEME_SNO_IFACE + lb_addresses: + - CHANGEME_SNO_CTLPLANE_LB_START-CHANGEME_SNO_CTLPLANE_LB_END + mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "ctlplane", + "plugins": [{ + "type": "macvlan", + "master": "ospbr", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_CTLPLANE_CIDR", + "range_start": "CHANGEME_SNO_CTLPLANE_NAD_START", + "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_CTLPLANE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" }, + { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" } + ] + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } + prefix-length: 24 + subnets: + - allocationRanges: + - end: CHANGEME_SNO_CTLPLANE_ALLOC1_END + start: CHANGEME_SNO_CTLPLANE_ALLOC1_START + - end: CHANGEME_SNO_CTLPLANE_ALLOC2_END + start: CHANGEME_SNO_CTLPLANE_ALLOC2_START + cidr: CHANGEME_SNO_CTLPLANE_CIDR + gateway: CHANGEME_SNO_CTLPLANE_GW + name: subnet1 + vlan: CHANGEME_SNO_CTLPLANE_VLAN + routes: + - destination: CHANGEME_EDPM_CTLPLANE_CIDR + nexthop: CHANGEME_SNO_CTLPLANE_GW + # Pods with only ctlplane attached (e.g. tempest) need this + # to reach Ceph RGW on the remote storage network. + - destination: CHANGEME_EDPM_STORAGE_CIDR + nexthop: CHANGEME_SNO_CTLPLANE_GW + # CHANGEME: site4 EDPM computes -- replace CIDRs/gateway/VLAN + - allocationRanges: + - end: CHANGEME_EDPM_CTLPLANE_END + start: CHANGEME_EDPM_CTLPLANE_START + cidr: CHANGEME_EDPM_CTLPLANE_CIDR + gateway: CHANGEME_EDPM_CTLPLANE_GW + name: subnet2 + vlan: CHANGEME_EDPM_CTLPLANE_VLAN + routes: + - destination: CHANGEME_SNO_CTLPLANE_CIDR + nexthop: CHANGEME_EDPM_CTLPLANE_GW + datacentre: + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "datacentre", + "plugins": [{ + "type": "host-device", + "device": "CHANGEME_SNO_IFACE", + "ipam": {} + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } + dns-resolver: + config: + search: [] + server: + - CHANGEME_SNO_DNS_SERVER + options: + - key: server + values: + - CHANGEME_SNO_DNS_SERVER + external: + base_iface: ospbr + dnsDomain: external.example.com + iface: external + mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "external", + "plugins": [{ + "type": "macvlan", + "master": "external", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_EXTERNAL_CIDR", + "range_start": "CHANGEME_SNO_EXTERNAL_NAD_START", + "range_end": "CHANGEME_SNO_EXTERNAL_NAD_END" + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } + prefix-length: 24 + subnets: + - allocationRanges: + - end: CHANGEME_SNO_EXTERNAL_END + start: CHANGEME_SNO_EXTERNAL_START + cidr: CHANGEME_SNO_EXTERNAL_CIDR + gateway: CHANGEME_SNO_EXTERNAL_GW + name: subnet1 + vlan: CHANGEME_SNO_EXTERNAL_VLAN + # CHANGEME: site4 external subnet + - allocationRanges: + - end: CHANGEME_EDPM_EXTERNAL_END + start: CHANGEME_EDPM_EXTERNAL_START + cidr: CHANGEME_EDPM_EXTERNAL_CIDR + gateway: CHANGEME_EDPM_EXTERNAL_GW + name: subnet2 + vlan: CHANGEME_EDPM_EXTERNAL_VLAN + vlan: CHANGEME_SNO_EXTERNAL_VLAN + internalapi: + base_iface: CHANGEME_SNO_IFACE + dnsDomain: internalapi.example.com + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_INTAPI_LB_IP + iface: internalapi + lb_addresses: + - CHANGEME_SNO_INTAPI_LB_START-CHANGEME_SNO_INTAPI_LB_END + mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "internalapi", + "plugins": [{ + "type": "macvlan", + "master": "internalapi", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_INTAPI_CIDR", + "range_start": "CHANGEME_SNO_INTAPI_NAD_START", + "range_end": "CHANGEME_SNO_INTAPI_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_INTAPI_CIDR", "gw": "CHANGEME_SNO_INTAPI_GW" } + ] + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } + prefix-length: 24 + subnets: + - allocationRanges: + - end: CHANGEME_SNO_INTAPI_END + start: CHANGEME_SNO_INTAPI_START + cidr: CHANGEME_SNO_INTAPI_CIDR + gateway: CHANGEME_SNO_INTAPI_GW + name: subnet1 + vlan: CHANGEME_SNO_INTAPI_VLAN + routes: + - destination: CHANGEME_EDPM_INTAPI_CIDR + nexthop: CHANGEME_SNO_INTAPI_GW + # CHANGEME: site4 internalapi subnet + - allocationRanges: + - end: CHANGEME_EDPM_INTAPI_END + start: CHANGEME_EDPM_INTAPI_START + cidr: CHANGEME_EDPM_INTAPI_CIDR + gateway: CHANGEME_EDPM_INTAPI_GW + name: subnet2 + vlan: CHANGEME_EDPM_INTAPI_VLAN + routes: + - destination: CHANGEME_SNO_INTAPI_CIDR + nexthop: CHANGEME_EDPM_INTAPI_GW + vlan: CHANGEME_SNO_INTAPI_VLAN + lbServiceType: LoadBalancer + # SNO: single OCP node only + node_0: + ctlplane_ip: CHANGEME_SNO_CTLPLANE_IP + external_ip: CHANGEME_SNO_EXTERNAL_IP + internalapi_ip: CHANGEME_SNO_INTAPI_IP + name: CHANGEME_SNO_NODE_NAME + storage_ip: CHANGEME_SNO_STORAGE_IP + # storagemgmt_ip: CHANGEME_SNO_STGMGMT_IP + tenant_ip: CHANGEME_SNO_TENANT_IP + # SNO: node_1/node_2 are required by lib/nncp replacement sources + # even though the NNCP resources are deleted by $patch:delete. + node_1: + ctlplane_ip: _unused_ + internalapi_ip: _unused_ + name: node-1 + storage_ip: _unused_ + tenant_ip: _unused_ + node_2: + ctlplane_ip: _unused_ + internalapi_ip: _unused_ + name: node-2 + storage_ip: _unused_ + tenant_ip: _unused_ + rabbitmq: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_RABBITMQ_IP + rabbitmq-cell1: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_RABBITMQ_CELL1_IP + routes: + config: + # CHANGEME: host-level routes for cross-site reachability (EDPM subnet2) + - destination: CHANGEME_EDPM_CTLPLANE_CIDR + next-hop-address: CHANGEME_SNO_CTLPLANE_GW + next-hop-interface: ospbr + table-id: 254 + - destination: CHANGEME_EDPM_INTAPI_CIDR + next-hop-address: CHANGEME_SNO_INTAPI_GW + next-hop-interface: internalapi + table-id: 254 + - destination: CHANGEME_EDPM_STORAGE_CIDR + next-hop-address: CHANGEME_SNO_STORAGE_GW + next-hop-interface: storage + table-id: 254 + - destination: CHANGEME_EDPM_TENANT_CIDR + next-hop-address: CHANGEME_SNO_TENANT_GW + next-hop-interface: tenant + table-id: 254 + storage: + base_iface: CHANGEME_SNO_IFACE + dnsDomain: storage.example.com + endpoint_annotations: + metallb.universe.tf/address-pool: storage + metallb.universe.tf/allow-shared-ip: storage + metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_STORAGE_LB_IP + iface: storage + lb_addresses: + - CHANGEME_SNO_STORAGE_LB_START-CHANGEME_SNO_STORAGE_LB_END + mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "storage", + "plugins": [{ + "type": "macvlan", + "master": "storage", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_STORAGE_CIDR", + "range_start": "CHANGEME_SNO_STORAGE_NAD_START", + "range_end": "CHANGEME_SNO_STORAGE_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_STORAGE_GW" } + ] + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } + prefix-length: 24 + subnets: + - allocationRanges: + - end: CHANGEME_SNO_STORAGE_END + start: CHANGEME_SNO_STORAGE_START + cidr: CHANGEME_SNO_STORAGE_CIDR + gateway: CHANGEME_SNO_STORAGE_GW + name: subnet1 + vlan: CHANGEME_SNO_STORAGE_VLAN + routes: + - destination: CHANGEME_EDPM_STORAGE_CIDR + nexthop: CHANGEME_SNO_STORAGE_GW + # CHANGEME: site4 storage subnet + - allocationRanges: + - end: CHANGEME_EDPM_STORAGE_END + start: CHANGEME_EDPM_STORAGE_START + cidr: CHANGEME_EDPM_STORAGE_CIDR + gateway: CHANGEME_EDPM_STORAGE_GW + name: subnet2 + vlan: CHANGEME_EDPM_STORAGE_VLAN + routes: + - destination: CHANGEME_SNO_STORAGE_CIDR + nexthop: CHANGEME_EDPM_STORAGE_GW + vlan: CHANGEME_SNO_STORAGE_VLAN + storageClass: lvms-local-storage + storagemgmt: + base_iface: CHANGEME_SNO_IFACE + dnsDomain: storagemgmt.example.com + iface: storagemgmt + lb_addresses: + - CHANGEME_SNO_STGMGMT_LB_START-CHANGEME_SNO_STGMGMT_LB_END + mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "storagemgmt", + "plugins": [{ + "type": "macvlan", + "master": "storagemgmt", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_STGMGMT_CIDR", + "range_start": "CHANGEME_SNO_STGMGMT_NAD_START", + "range_end": "CHANGEME_SNO_STGMGMT_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_STGMGMT_CIDR", "gw": "CHANGEME_SNO_STGMGMT_GW" } + ] + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } + prefix-length: 24 + subnets: + - allocationRanges: + - end: CHANGEME_SNO_STGMGMT_END + start: CHANGEME_SNO_STGMGMT_START + cidr: CHANGEME_SNO_STGMGMT_CIDR + gateway: CHANGEME_SNO_STGMGMT_GW + name: subnet1 + vlan: CHANGEME_SNO_STGMGMT_VLAN + routes: + - destination: CHANGEME_EDPM_STGMGMT_CIDR + nexthop: CHANGEME_SNO_STGMGMT_GW + # CHANGEME: site4 storagemgmt subnet + - allocationRanges: + - end: CHANGEME_EDPM_STGMGMT_END + start: CHANGEME_EDPM_STGMGMT_START + cidr: CHANGEME_EDPM_STGMGMT_CIDR + gateway: CHANGEME_EDPM_STGMGMT_GW + name: subnet2 + vlan: CHANGEME_EDPM_STGMGMT_VLAN + routes: + - destination: CHANGEME_SNO_STGMGMT_CIDR + nexthop: CHANGEME_EDPM_STGMGMT_GW + vlan: CHANGEME_SNO_STGMGMT_VLAN + tenant: + base_iface: CHANGEME_SNO_IFACE + dnsDomain: tenant.example.com + endpoint_annotations: + metallb.universe.tf/address-pool: tenant + metallb.universe.tf/allow-shared-ip: tenant + metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_TENANT_LB_IP + iface: tenant + lb_addresses: + - CHANGEME_SNO_TENANT_LB_START-CHANGEME_SNO_TENANT_LB_END + mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "tenant", + "plugins": [{ + "type": "macvlan", + "master": "tenant", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_TENANT_CIDR", + "range_start": "CHANGEME_SNO_TENANT_NAD_START", + "range_end": "CHANGEME_SNO_TENANT_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_TENANT_CIDR", "gw": "CHANGEME_SNO_TENANT_GW" } + ] + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } + prefix-length: 24 + subnets: + - allocationRanges: + - end: CHANGEME_SNO_TENANT_END + start: CHANGEME_SNO_TENANT_START + cidr: CHANGEME_SNO_TENANT_CIDR + gateway: CHANGEME_SNO_TENANT_GW + name: subnet1 + vlan: CHANGEME_SNO_TENANT_VLAN + routes: + - destination: CHANGEME_EDPM_TENANT_CIDR + nexthop: CHANGEME_SNO_TENANT_GW + # CHANGEME: site4 tenant subnet + - allocationRanges: + - end: CHANGEME_EDPM_TENANT_END + start: CHANGEME_EDPM_TENANT_START + cidr: CHANGEME_EDPM_TENANT_CIDR + gateway: CHANGEME_EDPM_TENANT_GW + name: subnet2 + vlan: CHANGEME_EDPM_TENANT_VLAN + routes: + - destination: CHANGEME_SNO_TENANT_CIDR + nexthop: CHANGEME_EDPM_TENANT_GW + vlan: CHANGEME_SNO_TENANT_VLAN + +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/local-config: 'true' + name: network-values diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml new file mode 100644 index 000000000..e74585df4 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml @@ -0,0 +1,101 @@ +# local-config: referenced, but not emitted by kustomize +# +# Post-Ceph control plane update: adds Glance RBD backend, Ceph +# extraMounts, and preserves all fields from the initial control-plane +# stage that lib/control-plane/base would otherwise reset to defaults. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-values + annotations: + config.kubernetes.io/local-config: "true" +data: + preserveJobs: true + notificationsBus: + cluster: rabbitmq + tls: + caBundleSecretName: custom-ca-certs + podLevel: + enabled: true + neutron: + customServiceConfig: | + [ml2] + mechanism_drivers = ovn + nova: + apiServiceTemplate: + customServiceConfig: | + [pci] + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + [filter_scheduler] + pci_in_placement = True + cell0: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + cell1: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + telemetry: + enabled: true + template: + ceilometer: + enabled: true + logging: + enabled: false + metricStorage: + dashboardsEnabled: true + enabled: true + dataplaneNetwork: ctlplane + networkAttachments: + - ctlplane + monitoringStack: + alertingEnabled: true + alertmanagerConfig: + replicas: 1 + scrapeInterval: 30s + storage: + persistent: + pvcStorageClass: lvms-local-storage + pvcStorageRequest: 20G + pvcStorageSelector: {} + retention: 24h + strategy: persistent + swift: + enabled: false + glance: + default: + replicas: 1 + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:rbd + [glance_store] + default_backend = default_backend + [default_backend] + rbd_store_ceph_conf = /etc/ceph/ceph.conf + store_description = "Ceph RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + extraMounts: + - name: v1 + region: r1 + extraVol: + - propagation: + - GlanceAPI + extraVolType: Ceph + volumes: + - name: ceph + secret: + secretName: ceph-conf-files + mounts: + - name: ceph + mountPath: /etc/ceph + readOnly: true diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml new file mode 100644 index 000000000..c8bdfc336 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml @@ -0,0 +1,18 @@ +# local-config: referenced, but not emitted by kustomize +# +# Ceph config for the control-plane-post-ceph stage. +# In CI, ci_gen_kustomize_values populates ceph_conf from /tmp/ files +# written by the ceph.yml hook (via the edpm-nodeset-values-post-ceph +# Jinja2 template). For manual use, replace CHANGEME values with +# base64-encoded Ceph keyring and conf. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-nodeset-values-post-ceph + annotations: + config.kubernetes.io/local-config: "true" +data: + ceph_conf: + ceph.client.openstack.keyring: CHANGEME_CEPH_KEYRING + ceph.conf: CHANGEME_CEPH_CONF diff --git a/examples/dt/nova/nova05epsilon/control-plane.md b/examples/dt/nova/nova05epsilon/control-plane.md index d731c3f64..3f54955d7 100644 --- a/examples/dt/nova/nova05epsilon/control-plane.md +++ b/examples/dt/nova/nova05epsilon/control-plane.md @@ -113,7 +113,11 @@ oc -n openshift-dns wait dns.operator/default --for condition=Available --timeou ``` Now wait for the full control plane readiness before proceeding to -the data plane stages: +the data plane stages. Note: the initial control plane deploys +without Ceph configuration (Glance uses defaults). Ceph backend +and extraMounts are added in the +[control-plane-post-ceph](control-plane-post-ceph.md) stage after +Ceph is installed. ```shell oc wait osctlplane controlplane --for condition=Ready --timeout=600s diff --git a/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml b/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml index d531c5d0c..107e3bba9 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml @@ -25,19 +25,8 @@ data: nicMappings: datacentre: ospbr glance: - customServiceConfig: | - [DEFAULT] - enabled_backends = default_backend:rbd - [glance_store] - default_backend = default_backend - [default_backend] - rbd_store_ceph_conf = /etc/ceph/ceph.conf - store_description = "Ceph RBD backend" - rbd_store_pool = images - rbd_store_user = openstack - rbd_thin_provisioning = True default: - replicas: 1 + replicas: 0 swift: enabled: false telemetry: @@ -87,18 +76,5 @@ data: customServiceConfig: | [filter_scheduler] pci_in_placement = True - extraMounts: - - name: v1 - region: r1 - extraVol: - - propagation: - - GlanceAPI - extraVolType: Ceph - volumes: - - name: ceph - secret: - secretName: ceph-conf-files - mounts: - - name: ceph - mountPath: /etc/ceph - readOnly: true + # extraMounts for Ceph are applied during the post-ceph stage + # (see edpm-post-ceph/ kustomization) after ceph-conf-files secret exists. diff --git a/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md b/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md index 130215b33..c1f917424 100644 --- a/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md +++ b/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md @@ -1,9 +1,10 @@ -# Update the control plane and finish deploying the data plane after Ceph has been installed +# Finish deploying the data plane after Ceph has been installed ## Assumptions - The [pre-ceph data plane](dataplane-pre-ceph.md) has been deployed - Ceph has been installed on the compute nodes +- The [control plane has been updated with Ceph configuration](control-plane-post-ceph.md) ## Initialize @@ -19,9 +20,8 @@ Change to the nova05epsilon directory cd architecture/examples/dt/nova/nova05epsilon ``` -Edit the [values.yaml](values.yaml) and [service-values.yaml](service-values.yaml) -files to suit your environment. In particular, update the Ceph configuration -placeholders in `values.yaml`: +Edit the [values.yaml](values.yaml) file to suit your environment. +In particular, update the Ceph configuration placeholders: - **`data.ceph_conf`** (DCN convention): A dict mapping Ceph filenames to base64-encoded content. For a single-site deployment, use plain filenames. @@ -48,13 +48,11 @@ Replace `` with the Ceph cluster FSID (from `ceph fsid`). ```shell vi values.yaml -vi service-values.yaml ``` -## Update the control plane and deploy the post-ceph dataplane +## Deploy the post-ceph dataplane -Generate the post-ceph CRs (this includes both the updated control plane -and the post-ceph nodeset): +Generate the post-ceph dataplane CRs: ```shell kustomize build > dataplane-nodeset.yaml @@ -66,12 +64,6 @@ Apply the CRs: oc apply -f dataplane-nodeset.yaml ``` -Wait for control plane to be available: - -```shell -oc wait osctlplane controlplane --for condition=Ready --timeout=600s -``` - Wait for the nodeset setup to complete: ```shell diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml index 1e4826c1e..8d8bbf089 100644 --- a/zuul.d/validations.yaml +++ b/zuul.d/validations.yaml @@ -311,6 +311,7 @@ - automation/net-env/nova05epsilon.yaml - examples/dt/nova/nova05epsilon - examples/dt/nova/nova05epsilon/control-plane + - examples/dt/nova/nova05epsilon/control-plane-post-ceph - examples/dt/nova/nova05epsilon/control-plane/networking - examples/dt/nova/nova05epsilon/control-plane/networking/dns - examples/dt/nova/nova05epsilon/control-plane/networking/nncp From f39e475f29279fab9a53a60011e88684f98b5b8c Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Wed, 17 Jun 2026 13:44:18 +0200 Subject: [PATCH 5/5] nova05: deduplicate network-values for pre/post CP Signed-off-by: Bohdan Dobrelia --- automation/vars/nova05epsilon.yaml | 8 +- .../kustomization.yaml | 2 +- .../network-values.yaml | 429 ------------------ .../control-plane/kustomization.yaml | 2 +- .../networking/kustomization.yaml | 4 +- .../networking/nncp/kustomization.yaml | 5 +- .../dt/nova/nova05epsilon/kustomization.yaml | 2 +- .../network-values/kustomization.yaml | 6 + .../nncp => network-values}/values.yaml | 0 zuul.d/validations.yaml | 1 + 10 files changed, 16 insertions(+), 443 deletions(-) delete mode 100644 examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml create mode 100644 examples/dt/nova/nova05epsilon/network-values/kustomization.yaml rename examples/dt/nova/nova05epsilon/{control-plane/networking/nncp => network-values}/values.yaml (100%) diff --git a/automation/vars/nova05epsilon.yaml b/automation/vars/nova05epsilon.yaml index 3a87f0aab..5dd1503bf 100644 --- a/automation/vars/nova05epsilon.yaml +++ b/automation/vars/nova05epsilon.yaml @@ -12,7 +12,7 @@ vas: --timeout=5m values: - name: network-values - src_file: values.yaml + src_file: ../../../network-values/values.yaml build_output: nncp.yaml - name: network-configuration @@ -25,7 +25,7 @@ vas: --timeout=5m values: - name: network-values - src_file: nncp/values.yaml + src_file: ../../network-values/values.yaml build_output: network.yaml - name: control-plane @@ -37,7 +37,7 @@ vas: --timeout=20m values: - name: network-values - src_file: networking/nncp/values.yaml + src_file: ../network-values/values.yaml - name: service-values src_file: service-values.yaml build_output: control-plane.yaml @@ -110,7 +110,7 @@ vas: --timeout=20m values: - name: network-values - src_file: network-values.yaml + src_file: ../network-values/values.yaml - name: service-values src_file: service-values.yaml - name: edpm-nodeset-values-post-ceph diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml index 9d3f4689a..432094a6f 100644 --- a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml @@ -3,9 +3,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization components: + - ../network-values - ../../../../../dt/nova/nova05epsilon/control-plane-post-ceph resources: - - network-values.yaml - values.yaml - service-values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml deleted file mode 100644 index 66b1a66de..000000000 --- a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/network-values.yaml +++ /dev/null @@ -1,429 +0,0 @@ -# Network values for SNO + DCN spine-leaf topology. -# -# Site 2 (SNO control plane): subnet1 -# Site 4 (EDPM computes): subnet2 -# -# Replace all CHANGEME values to match your environment. -# All subnets must share a common routable address space. ---- -apiVersion: v1 -data: - bridgeName: ospbr - ctlplane: - dnsDomain: ctlplane.example.com - endpoint_annotations: - metallb.universe.tf/address-pool: ctlplane - metallb.universe.tf/allow-shared-ip: ctlplane - metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_CTLPLANE_LB_IP - iface: CHANGEME_SNO_IFACE - lb_addresses: - - CHANGEME_SNO_CTLPLANE_LB_START-CHANGEME_SNO_CTLPLANE_LB_END - mtu: 1500 - net-attach-def: | - { - "cniVersion": "0.4.0", - "name": "ctlplane", - "plugins": [{ - "type": "macvlan", - "master": "ospbr", - "ipam": { - "type": "whereabouts", - "range": "CHANGEME_SNO_CTLPLANE_CIDR", - "range_start": "CHANGEME_SNO_CTLPLANE_NAD_START", - "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END", - "routes": [ - { "dst": "CHANGEME_EDPM_CTLPLANE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" }, - { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" } - ] - } - }, { - "type": "tuning", - "sysctl": { - "net.ipv6.conf.IFNAME.accept_ra": "0" - } - }] - } - prefix-length: 24 - subnets: - - allocationRanges: - - end: CHANGEME_SNO_CTLPLANE_ALLOC1_END - start: CHANGEME_SNO_CTLPLANE_ALLOC1_START - - end: CHANGEME_SNO_CTLPLANE_ALLOC2_END - start: CHANGEME_SNO_CTLPLANE_ALLOC2_START - cidr: CHANGEME_SNO_CTLPLANE_CIDR - gateway: CHANGEME_SNO_CTLPLANE_GW - name: subnet1 - vlan: CHANGEME_SNO_CTLPLANE_VLAN - routes: - - destination: CHANGEME_EDPM_CTLPLANE_CIDR - nexthop: CHANGEME_SNO_CTLPLANE_GW - # Pods with only ctlplane attached (e.g. tempest) need this - # to reach Ceph RGW on the remote storage network. - - destination: CHANGEME_EDPM_STORAGE_CIDR - nexthop: CHANGEME_SNO_CTLPLANE_GW - # CHANGEME: site4 EDPM computes -- replace CIDRs/gateway/VLAN - - allocationRanges: - - end: CHANGEME_EDPM_CTLPLANE_END - start: CHANGEME_EDPM_CTLPLANE_START - cidr: CHANGEME_EDPM_CTLPLANE_CIDR - gateway: CHANGEME_EDPM_CTLPLANE_GW - name: subnet2 - vlan: CHANGEME_EDPM_CTLPLANE_VLAN - routes: - - destination: CHANGEME_SNO_CTLPLANE_CIDR - nexthop: CHANGEME_EDPM_CTLPLANE_GW - datacentre: - net-attach-def: | - { - "cniVersion": "0.4.0", - "name": "datacentre", - "plugins": [{ - "type": "host-device", - "device": "CHANGEME_SNO_IFACE", - "ipam": {} - }, { - "type": "tuning", - "sysctl": { - "net.ipv6.conf.IFNAME.accept_ra": "0" - } - }] - } - dns-resolver: - config: - search: [] - server: - - CHANGEME_SNO_DNS_SERVER - options: - - key: server - values: - - CHANGEME_SNO_DNS_SERVER - external: - base_iface: ospbr - dnsDomain: external.example.com - iface: external - mtu: 1500 - net-attach-def: | - { - "cniVersion": "0.4.0", - "name": "external", - "plugins": [{ - "type": "macvlan", - "master": "external", - "mode": "bridge", - "ipam": { - "type": "whereabouts", - "range": "CHANGEME_SNO_EXTERNAL_CIDR", - "range_start": "CHANGEME_SNO_EXTERNAL_NAD_START", - "range_end": "CHANGEME_SNO_EXTERNAL_NAD_END" - } - }, { - "type": "tuning", - "sysctl": { - "net.ipv6.conf.IFNAME.accept_ra": "0" - } - }] - } - prefix-length: 24 - subnets: - - allocationRanges: - - end: CHANGEME_SNO_EXTERNAL_END - start: CHANGEME_SNO_EXTERNAL_START - cidr: CHANGEME_SNO_EXTERNAL_CIDR - gateway: CHANGEME_SNO_EXTERNAL_GW - name: subnet1 - vlan: CHANGEME_SNO_EXTERNAL_VLAN - # CHANGEME: site4 external subnet - - allocationRanges: - - end: CHANGEME_EDPM_EXTERNAL_END - start: CHANGEME_EDPM_EXTERNAL_START - cidr: CHANGEME_EDPM_EXTERNAL_CIDR - gateway: CHANGEME_EDPM_EXTERNAL_GW - name: subnet2 - vlan: CHANGEME_EDPM_EXTERNAL_VLAN - vlan: CHANGEME_SNO_EXTERNAL_VLAN - internalapi: - base_iface: CHANGEME_SNO_IFACE - dnsDomain: internalapi.example.com - endpoint_annotations: - metallb.universe.tf/address-pool: internalapi - metallb.universe.tf/allow-shared-ip: internalapi - metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_INTAPI_LB_IP - iface: internalapi - lb_addresses: - - CHANGEME_SNO_INTAPI_LB_START-CHANGEME_SNO_INTAPI_LB_END - mtu: 1500 - net-attach-def: | - { - "cniVersion": "0.4.0", - "name": "internalapi", - "plugins": [{ - "type": "macvlan", - "master": "internalapi", - "ipam": { - "type": "whereabouts", - "range": "CHANGEME_SNO_INTAPI_CIDR", - "range_start": "CHANGEME_SNO_INTAPI_NAD_START", - "range_end": "CHANGEME_SNO_INTAPI_NAD_END", - "routes": [ - { "dst": "CHANGEME_EDPM_INTAPI_CIDR", "gw": "CHANGEME_SNO_INTAPI_GW" } - ] - } - }, { - "type": "tuning", - "sysctl": { - "net.ipv6.conf.IFNAME.accept_ra": "0" - } - }] - } - prefix-length: 24 - subnets: - - allocationRanges: - - end: CHANGEME_SNO_INTAPI_END - start: CHANGEME_SNO_INTAPI_START - cidr: CHANGEME_SNO_INTAPI_CIDR - gateway: CHANGEME_SNO_INTAPI_GW - name: subnet1 - vlan: CHANGEME_SNO_INTAPI_VLAN - routes: - - destination: CHANGEME_EDPM_INTAPI_CIDR - nexthop: CHANGEME_SNO_INTAPI_GW - # CHANGEME: site4 internalapi subnet - - allocationRanges: - - end: CHANGEME_EDPM_INTAPI_END - start: CHANGEME_EDPM_INTAPI_START - cidr: CHANGEME_EDPM_INTAPI_CIDR - gateway: CHANGEME_EDPM_INTAPI_GW - name: subnet2 - vlan: CHANGEME_EDPM_INTAPI_VLAN - routes: - - destination: CHANGEME_SNO_INTAPI_CIDR - nexthop: CHANGEME_EDPM_INTAPI_GW - vlan: CHANGEME_SNO_INTAPI_VLAN - lbServiceType: LoadBalancer - # SNO: single OCP node only - node_0: - ctlplane_ip: CHANGEME_SNO_CTLPLANE_IP - external_ip: CHANGEME_SNO_EXTERNAL_IP - internalapi_ip: CHANGEME_SNO_INTAPI_IP - name: CHANGEME_SNO_NODE_NAME - storage_ip: CHANGEME_SNO_STORAGE_IP - # storagemgmt_ip: CHANGEME_SNO_STGMGMT_IP - tenant_ip: CHANGEME_SNO_TENANT_IP - # SNO: node_1/node_2 are required by lib/nncp replacement sources - # even though the NNCP resources are deleted by $patch:delete. - node_1: - ctlplane_ip: _unused_ - internalapi_ip: _unused_ - name: node-1 - storage_ip: _unused_ - tenant_ip: _unused_ - node_2: - ctlplane_ip: _unused_ - internalapi_ip: _unused_ - name: node-2 - storage_ip: _unused_ - tenant_ip: _unused_ - rabbitmq: - endpoint_annotations: - metallb.universe.tf/address-pool: internalapi - metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_RABBITMQ_IP - rabbitmq-cell1: - endpoint_annotations: - metallb.universe.tf/address-pool: internalapi - metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_RABBITMQ_CELL1_IP - routes: - config: - # CHANGEME: host-level routes for cross-site reachability (EDPM subnet2) - - destination: CHANGEME_EDPM_CTLPLANE_CIDR - next-hop-address: CHANGEME_SNO_CTLPLANE_GW - next-hop-interface: ospbr - table-id: 254 - - destination: CHANGEME_EDPM_INTAPI_CIDR - next-hop-address: CHANGEME_SNO_INTAPI_GW - next-hop-interface: internalapi - table-id: 254 - - destination: CHANGEME_EDPM_STORAGE_CIDR - next-hop-address: CHANGEME_SNO_STORAGE_GW - next-hop-interface: storage - table-id: 254 - - destination: CHANGEME_EDPM_TENANT_CIDR - next-hop-address: CHANGEME_SNO_TENANT_GW - next-hop-interface: tenant - table-id: 254 - storage: - base_iface: CHANGEME_SNO_IFACE - dnsDomain: storage.example.com - endpoint_annotations: - metallb.universe.tf/address-pool: storage - metallb.universe.tf/allow-shared-ip: storage - metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_STORAGE_LB_IP - iface: storage - lb_addresses: - - CHANGEME_SNO_STORAGE_LB_START-CHANGEME_SNO_STORAGE_LB_END - mtu: 1500 - net-attach-def: | - { - "cniVersion": "0.4.0", - "name": "storage", - "plugins": [{ - "type": "macvlan", - "master": "storage", - "ipam": { - "type": "whereabouts", - "range": "CHANGEME_SNO_STORAGE_CIDR", - "range_start": "CHANGEME_SNO_STORAGE_NAD_START", - "range_end": "CHANGEME_SNO_STORAGE_NAD_END", - "routes": [ - { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_STORAGE_GW" } - ] - } - }, { - "type": "tuning", - "sysctl": { - "net.ipv6.conf.IFNAME.accept_ra": "0" - } - }] - } - prefix-length: 24 - subnets: - - allocationRanges: - - end: CHANGEME_SNO_STORAGE_END - start: CHANGEME_SNO_STORAGE_START - cidr: CHANGEME_SNO_STORAGE_CIDR - gateway: CHANGEME_SNO_STORAGE_GW - name: subnet1 - vlan: CHANGEME_SNO_STORAGE_VLAN - routes: - - destination: CHANGEME_EDPM_STORAGE_CIDR - nexthop: CHANGEME_SNO_STORAGE_GW - # CHANGEME: site4 storage subnet - - allocationRanges: - - end: CHANGEME_EDPM_STORAGE_END - start: CHANGEME_EDPM_STORAGE_START - cidr: CHANGEME_EDPM_STORAGE_CIDR - gateway: CHANGEME_EDPM_STORAGE_GW - name: subnet2 - vlan: CHANGEME_EDPM_STORAGE_VLAN - routes: - - destination: CHANGEME_SNO_STORAGE_CIDR - nexthop: CHANGEME_EDPM_STORAGE_GW - vlan: CHANGEME_SNO_STORAGE_VLAN - storageClass: lvms-local-storage - storagemgmt: - base_iface: CHANGEME_SNO_IFACE - dnsDomain: storagemgmt.example.com - iface: storagemgmt - lb_addresses: - - CHANGEME_SNO_STGMGMT_LB_START-CHANGEME_SNO_STGMGMT_LB_END - mtu: 1500 - net-attach-def: | - { - "cniVersion": "0.4.0", - "name": "storagemgmt", - "plugins": [{ - "type": "macvlan", - "master": "storagemgmt", - "ipam": { - "type": "whereabouts", - "range": "CHANGEME_SNO_STGMGMT_CIDR", - "range_start": "CHANGEME_SNO_STGMGMT_NAD_START", - "range_end": "CHANGEME_SNO_STGMGMT_NAD_END", - "routes": [ - { "dst": "CHANGEME_EDPM_STGMGMT_CIDR", "gw": "CHANGEME_SNO_STGMGMT_GW" } - ] - } - }, { - "type": "tuning", - "sysctl": { - "net.ipv6.conf.IFNAME.accept_ra": "0" - } - }] - } - prefix-length: 24 - subnets: - - allocationRanges: - - end: CHANGEME_SNO_STGMGMT_END - start: CHANGEME_SNO_STGMGMT_START - cidr: CHANGEME_SNO_STGMGMT_CIDR - gateway: CHANGEME_SNO_STGMGMT_GW - name: subnet1 - vlan: CHANGEME_SNO_STGMGMT_VLAN - routes: - - destination: CHANGEME_EDPM_STGMGMT_CIDR - nexthop: CHANGEME_SNO_STGMGMT_GW - # CHANGEME: site4 storagemgmt subnet - - allocationRanges: - - end: CHANGEME_EDPM_STGMGMT_END - start: CHANGEME_EDPM_STGMGMT_START - cidr: CHANGEME_EDPM_STGMGMT_CIDR - gateway: CHANGEME_EDPM_STGMGMT_GW - name: subnet2 - vlan: CHANGEME_EDPM_STGMGMT_VLAN - routes: - - destination: CHANGEME_SNO_STGMGMT_CIDR - nexthop: CHANGEME_EDPM_STGMGMT_GW - vlan: CHANGEME_SNO_STGMGMT_VLAN - tenant: - base_iface: CHANGEME_SNO_IFACE - dnsDomain: tenant.example.com - endpoint_annotations: - metallb.universe.tf/address-pool: tenant - metallb.universe.tf/allow-shared-ip: tenant - metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_TENANT_LB_IP - iface: tenant - lb_addresses: - - CHANGEME_SNO_TENANT_LB_START-CHANGEME_SNO_TENANT_LB_END - mtu: 1500 - net-attach-def: | - { - "cniVersion": "0.4.0", - "name": "tenant", - "plugins": [{ - "type": "macvlan", - "master": "tenant", - "ipam": { - "type": "whereabouts", - "range": "CHANGEME_SNO_TENANT_CIDR", - "range_start": "CHANGEME_SNO_TENANT_NAD_START", - "range_end": "CHANGEME_SNO_TENANT_NAD_END", - "routes": [ - { "dst": "CHANGEME_EDPM_TENANT_CIDR", "gw": "CHANGEME_SNO_TENANT_GW" } - ] - } - }, { - "type": "tuning", - "sysctl": { - "net.ipv6.conf.IFNAME.accept_ra": "0" - } - }] - } - prefix-length: 24 - subnets: - - allocationRanges: - - end: CHANGEME_SNO_TENANT_END - start: CHANGEME_SNO_TENANT_START - cidr: CHANGEME_SNO_TENANT_CIDR - gateway: CHANGEME_SNO_TENANT_GW - name: subnet1 - vlan: CHANGEME_SNO_TENANT_VLAN - routes: - - destination: CHANGEME_EDPM_TENANT_CIDR - nexthop: CHANGEME_SNO_TENANT_GW - # CHANGEME: site4 tenant subnet - - allocationRanges: - - end: CHANGEME_EDPM_TENANT_END - start: CHANGEME_EDPM_TENANT_START - cidr: CHANGEME_EDPM_TENANT_CIDR - gateway: CHANGEME_EDPM_TENANT_GW - name: subnet2 - vlan: CHANGEME_EDPM_TENANT_VLAN - routes: - - destination: CHANGEME_SNO_TENANT_CIDR - nexthop: CHANGEME_EDPM_TENANT_GW - vlan: CHANGEME_SNO_TENANT_VLAN - -kind: ConfigMap -metadata: - annotations: - config.kubernetes.io/local-config: 'true' - name: network-values diff --git a/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml index 127d1b018..81be4a0fc 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml @@ -3,8 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization components: + - ../network-values - ../../../../../dt/nova/nova05epsilon/ resources: - - networking/nncp/values.yaml - service-values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml index aabb8ba24..ab672f640 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml @@ -3,7 +3,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization components: + - ../../network-values - ../../../../../../dt/nova/nova05epsilon/networking - -resources: - - nncp/values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml index 4006234f5..6d0b2d85e 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml @@ -18,12 +18,9 @@ transformers: create: true components: + - ../../../network-values - ../../../../../../../dt/nova/nova05epsilon/networking/nncp # To expose storagemgmt VLAN on the SNO node, uncomment the patch and # replacements block in dt/nova/nova05epsilon/networking/nncp/kustomization.yaml # and uncomment node_0.storagemgmt_ip in values.yaml. # Not required for CephHCI (Ceph uses the storage network for OSD traffic). - - -resources: - - values.yaml diff --git a/examples/dt/nova/nova05epsilon/kustomization.yaml b/examples/dt/nova/nova05epsilon/kustomization.yaml index b4590d7e7..e08a55e7c 100644 --- a/examples/dt/nova/nova05epsilon/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/kustomization.yaml @@ -4,10 +4,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization components: + - network-values - ../../../../dt/nova/nova05epsilon/edpm-post-ceph/nodeset resources: - - control-plane/networking/nncp/values.yaml - edpm-pre-ceph/nodeset/values.yaml - service-values.yaml - values.yaml diff --git a/examples/dt/nova/nova05epsilon/network-values/kustomization.yaml b/examples/dt/nova/nova05epsilon/network-values/kustomization.yaml new file mode 100644 index 000000000..ae3071458 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/network-values/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml b/examples/dt/nova/nova05epsilon/network-values/values.yaml similarity index 100% rename from examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml rename to examples/dt/nova/nova05epsilon/network-values/values.yaml diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml index 8d8bbf089..72a1620d5 100644 --- a/zuul.d/validations.yaml +++ b/zuul.d/validations.yaml @@ -310,6 +310,7 @@ - automation/mocks/nova05epsilon.yaml - automation/net-env/nova05epsilon.yaml - examples/dt/nova/nova05epsilon + - examples/dt/nova/nova05epsilon/network-values - examples/dt/nova/nova05epsilon/control-plane - examples/dt/nova/nova05epsilon/control-plane-post-ceph - examples/dt/nova/nova05epsilon/control-plane/networking