diff --git a/automation/mocks/nova05epsilon.yaml b/automation/mocks/nova05epsilon.yaml new file mode 100644 index 000000000..654294e64 --- /dev/null +++ b/automation/mocks/nova05epsilon.yaml @@ -0,0 +1,42 @@ +--- +cifmw_install_ca_url: http://example.com/example.pem +cifmw_repo_setup_rhos_release_rpm: http://example.com/rhos-release.rpm +cifmw_repo_setup_rhos_release_args: example-args +cifmw_ci_gen_kustomize_values_remove_keys_expressions: + - ^node(_[0-9]+)?$ +cifmw_baremetal_hosts: + compute-0: + connection: idrac-virtualmedia://foo.example.com/redfish/v1/Systems/System.Embedded.1 + boot_mode: UEFI + status: running + nics: + - mac: "aa:bb:cc:dd:ee:ff" + network: provision + root_device_hint: /dev/disk/by-path/pci-0000:65:00.0-scsi-0:1:234:5 + root_device_hint_field: deviceName + credentials_file: "/home/zuul/idrac_access.yaml" + nmstate: + interfaces: + - name: nic1 + type: ethernet + state: up + ipv4: + enabled: true + dhcp: true + - name: nic2 + type: ethernet + state: up + ipv4: + enabled: true + address: + - ip: 10.0.0.10 + prefix-length: 24 + dns-resolver: + config: + server: + - 10.0.0.1 + routes: + config: + - destination: 0.0.0.0/0 + next-hop-address: 10.0.0.1 + next-hop-interface: nic1 diff --git a/automation/net-env/nova05epsilon.yaml b/automation/net-env/nova05epsilon.yaml new file mode 100644 index 000000000..89a9ebc1d --- /dev/null +++ b/automation/net-env/nova05epsilon.yaml @@ -0,0 +1,477 @@ +--- +instances: + compute-0: + name: compute-0 + networks: + ctlplane: + interface_name: eth1 + ip_v4: 192.168.122.100 + is_trunk_parent: true + mac_addr: 52:54:03:b1:78:6c + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: eth1.20 + ip_v4: 172.17.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:1e:58:da + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + storage: + interface_name: eth1.21 + ip_v4: 172.18.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:7e:be:ad + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + storagemgmt: + interface_name: eth1.23 + ip_v4: 172.20.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:aa:bb:cc + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storagemgmt + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 23 + tenant: + interface_name: eth1.22 + ip_v4: 172.19.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:33:ee:37 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + external: + interface_name: eth1.44 + ip_v4: 10.0.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:dd:ee:ff + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: external + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 44 + ocp-master-0: + hostname: master-0 + name: ocp-master-0 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.10 + is_trunk_parent: true + mac_addr: 52:54:00:15:07:f4 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:68:4b:ce + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:05:23:2c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + storagemgmt: + interface_name: enp6s0.23 + ip_v4: 172.20.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:11:22:33 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storagemgmt + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 23 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:06:80:f0 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + external: + interface_name: enp6s0.44 + ip_v4: 10.0.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:44:55:66 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: external + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 44 + # unused but required by lib components + ocp-master-1: + hostname: master-1 + name: ocp-master-1 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.11 + is_trunk_parent: true + mac_addr: 52:54:01:43:50:83 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:71:78:e6 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.11 + mac_addr: 52:54:01:51:68:1e + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:76:3d:ba + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:7a:5d:1d + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-2: + hostname: master-2 + name: ocp-master-2 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.12 + is_trunk_parent: true + mac_addr: 52:54:02:fd:b8:5a + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:66:a0:7c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.12 + mac_addr: 52:54:02:50:e9:87 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:09:45:3b + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:22:70:9f + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + +networks: + ctlplane: + dns_v4: + - 192.168.122.1 + dns_v6: [] + gw_v4: 192.168.122.1 + mtu: 1500 + network_name: ctlplane + network_v4: 192.168.122.0/24 + search_domain: ctlplane.example.com + tools: + metallb: + ipv4_ranges: + - end: 192.168.122.90 + end_host: 90 + length: 11 + start: 192.168.122.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 192.168.122.70 + end_host: 70 + length: 41 + start: 192.168.122.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 192.168.122.120 + end_host: 120 + length: 21 + start: 192.168.122.100 + start_host: 100 + - end: 192.168.122.170 + end_host: 170 + length: 21 + start: 192.168.122.150 + start_host: 150 + ipv6_ranges: [] + external: + dns_v4: [] + dns_v6: [] + mtu: 1500 + network_name: external + network_v4: 10.0.0.0/24 + search_domain: external.example.com + tools: + multus: + ipv4_ranges: + - end: 10.0.0.70 + end_host: 70 + length: 41 + start: 10.0.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 10.0.0.250 + end_host: 250 + length: 151 + start: 10.0.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 44 + internalapi: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: internalapi + network_v4: 172.17.0.0/24 + search_domain: internalapi.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.17.0.90 + end_host: 90 + length: 11 + start: 172.17.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.17.0.70 + end_host: 70 + length: 41 + start: 172.17.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.17.0.250 + end_host: 250 + length: 151 + start: 172.17.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 20 + storage: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: storage + network_v4: 172.18.0.0/24 + search_domain: storage.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.18.0.90 + end_host: 90 + length: 11 + start: 172.18.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.18.0.70 + end_host: 70 + length: 41 + start: 172.18.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.18.0.250 + end_host: 250 + length: 151 + start: 172.18.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 21 + storagemgmt: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: storagemgmt + network_v4: 172.20.0.0/24 + search_domain: storagemgmt.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.20.0.90 + end_host: 90 + length: 11 + start: 172.20.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.20.0.70 + end_host: 70 + length: 41 + start: 172.20.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.20.0.250 + end_host: 250 + length: 151 + start: 172.20.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 23 + tenant: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: tenant + network_v4: 172.19.0.0/24 + search_domain: tenant.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.19.0.90 + end_host: 90 + length: 11 + start: 172.19.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.19.0.70 + end_host: 70 + length: 41 + start: 172.19.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.19.0.250 + end_host: 250 + length: 151 + start: 172.19.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 22 +routers: {} diff --git a/automation/vars/nova05epsilon.yaml b/automation/vars/nova05epsilon.yaml index c3b507269..5dd1503bf 100644 --- a/automation/vars/nova05epsilon.yaml +++ b/automation/vars/nova05epsilon.yaml @@ -12,7 +12,7 @@ vas: --timeout=5m values: - name: network-values - src_file: values.yaml + src_file: ../../../network-values/values.yaml build_output: nncp.yaml - name: network-configuration @@ -25,7 +25,7 @@ vas: --timeout=5m values: - name: network-values - src_file: nncp/values.yaml + src_file: ../../network-values/values.yaml build_output: network.yaml - name: control-plane @@ -37,7 +37,7 @@ vas: --timeout=20m values: - name: network-values - src_file: networking/nncp/values.yaml + src_file: ../network-values/values.yaml - name: service-values src_file: service-values.yaml build_output: control-plane.yaml @@ -101,14 +101,25 @@ vas: type: playbook source: "../../hooks/playbooks/ceph.yml" inventory: "${HOME}/ci-framework-data/artifacts/ceph_inventory.yml" - - - name: edpm-nodeset-post-ceph - path: examples/dt/nova/nova05epsilon + - name: control-plane-post-ceph + path: examples/dt/nova/nova05epsilon/control-plane-post-ceph wait_conditions: - >- oc -n openstack wait osctlplane controlplane --for condition=Ready --timeout=20m + values: + - name: network-values + src_file: ../network-values/values.yaml + - name: service-values + src_file: service-values.yaml + - name: edpm-nodeset-values-post-ceph + src_file: values.yaml + build_output: control-plane-post-ceph.yaml + + - name: edpm-nodeset-post-ceph + path: examples/dt/nova/nova05epsilon + wait_conditions: - >- oc -n openstack wait osdpns gpu-computes-edpm diff --git a/dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml b/dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml new file mode 100644 index 000000000..4d2073239 --- /dev/null +++ b/dt/nova/nova05epsilon/control-plane-post-ceph/ceph_secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +data: + ceph.client.openstack.keyring: _ignored_ + ceph.conf: _ignored_ +kind: Secret +metadata: + name: ceph-conf-files + namespace: openstack +type: Opaque diff --git a/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml b/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml new file mode 100644 index 000000000..756080b46 --- /dev/null +++ b/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml @@ -0,0 +1,160 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../lib/control-plane/base + - ../../../../lib/control-plane/service-endpoints + - ../../../../lib/control-plane/dns + - ../../../../lib/control-plane/storage + - ../../../../lib/control-plane/ovn-bridge + - ../../../../lib/control-plane/job-settings + - ../../../../lib/control-plane/tls + - ../../../../lib/control-plane/messaging-bus + +resources: + - ceph_secret.yaml + +replacements: + # Ceph config files -> Secret + - source: + kind: ConfigMap + name: edpm-nodeset-values-post-ceph + fieldPath: data.ceph_conf + targets: + - select: + kind: Secret + name: ceph-conf-files + fieldPaths: + - data + options: + create: true + # Glance RBD backend + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.default.replicas + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.glanceAPIs.default.replicas + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.extraMounts + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.extraMounts + options: + create: true + # Swift + - source: + kind: ConfigMap + name: service-values + fieldPath: data.swift.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.swift.enabled + options: + create: true + # Telemetry + - source: + kind: ConfigMap + name: service-values + fieldPath: data.telemetry + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.telemetry + options: + create: true + # Neutron ML2 config + - source: + kind: ConfigMap + name: service-values + fieldPath: data.neutron.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.neutron.template.customServiceConfig + options: + create: true + # Nova PCI passthrough / scheduler config + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.apiServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.apiServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell0.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell0.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell1.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell1.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.schedulerServiceTemplate.customServiceConfig + options: + create: true diff --git a/dt/nova/nova05epsilon/control-plane/kustomization.yaml b/dt/nova/nova05epsilon/control-plane/kustomization.yaml index 7a69f6f50..b74de7e00 100644 --- a/dt/nova/nova05epsilon/control-plane/kustomization.yaml +++ b/dt/nova/nova05epsilon/control-plane/kustomization.yaml @@ -18,132 +18,4 @@ transformers: create: true components: - - ../../../../lib/control-plane/base - - ../../../../lib/control-plane/service-endpoints - - ../../../../lib/control-plane/dns - - ../../../../lib/control-plane/storage - - ../../../../lib/control-plane/ovn-bridge - - ../../../../lib/control-plane/job-settings - -replacements: - - source: - kind: ConfigMap - name: service-values - fieldPath: data.neutron.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.neutron.template.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.ovn.ovnController.nicMappings - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.ovn.template.ovnController.nicMappings - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.glance.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.glance.template.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.glance.default.replicas - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.glance.template.glanceAPIs.default.replicas - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.swift.enabled - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.swift.enabled - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.telemetry - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.telemetry - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.apiServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.apiServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.cell0.conductorServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.cellTemplates.cell0.conductorServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.cell1.conductorServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.cellTemplates.cell1.conductorServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.nova.template.schedulerServiceTemplate.customServiceConfig - options: - create: true - - source: - kind: ConfigMap - name: service-values - fieldPath: data.extraMounts - targets: - - select: - kind: OpenStackControlPlane - fieldPaths: - - spec.extraMounts - options: - create: true + - ../../../../lib/control-plane diff --git a/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml b/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml index 70ab2f823..e14426ab0 100644 --- a/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml +++ b/dt/nova/nova05epsilon/edpm-post-ceph/nodeset/kustomization.yaml @@ -18,7 +18,6 @@ transformers: create: true components: - - ../../control-plane/ - ../../../../../lib/dataplane/nodeset resources: diff --git a/dt/nova/nova05epsilon/kustomization.yaml b/dt/nova/nova05epsilon/kustomization.yaml index 2e88f79fb..014f43f6e 100644 --- a/dt/nova/nova05epsilon/kustomization.yaml +++ b/dt/nova/nova05epsilon/kustomization.yaml @@ -19,3 +19,104 @@ transformers: components: - ../../../lib/control-plane + +replacements: + - source: + kind: ConfigMap + name: service-values + fieldPath: data.neutron.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.neutron.template.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.ovn.ovnController.nicMappings + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.ovn.template.ovnController.nicMappings + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.default.replicas + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.glanceAPIs.default.replicas + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.swift.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.swift.enabled + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.telemetry + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.telemetry + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.apiServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.apiServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell0.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell0.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell1.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell1.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.schedulerServiceTemplate.customServiceConfig + options: + create: true diff --git a/dt/nova/nova05epsilon/networking/external-nad.yaml b/dt/nova/nova05epsilon/networking/external-nad.yaml new file mode 100644 index 000000000..65773c9d1 --- /dev/null +++ b/dt/nova/nova05epsilon/networking/external-nad.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: external + labels: + osp/net: external + osp/net-attach-def-type: standard diff --git a/dt/nova/nova05epsilon/networking/kustomization.yaml b/dt/nova/nova05epsilon/networking/kustomization.yaml index 66eadf45b..0d6ec1126 100644 --- a/dt/nova/nova05epsilon/networking/kustomization.yaml +++ b/dt/nova/nova05epsilon/networking/kustomization.yaml @@ -25,6 +25,7 @@ components: resources: - storagemgmt-nad.yaml - storagemgmt-metallb.yaml + - external-nad.yaml # Add storagemgmt network template, as it is needed for CephHCI patches: @@ -122,3 +123,15 @@ replacements: name: ctlplane fieldPaths: - spec.interfaces.0 + + # External NAD for provider network FIP access + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.net-attach-def + targets: + - select: + kind: NetworkAttachmentDefinition + name: external + fieldPaths: + - spec.config diff --git a/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml b/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml index 50cfe56e4..e7dbb1331 100644 --- a/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml +++ b/dt/nova/nova05epsilon/networking/nncp/kustomization.yaml @@ -40,93 +40,92 @@ patches: kind: NodeNetworkConfigurationPolicy metadata: name: node-2 - # SNO: remove worker scheduling label from nodeSelector + # SNO: remove worker scheduling label from nodeSelector. + # Use labelSelector because lib/nncp renames the resource before + # this component's patches run. - target: kind: NodeNetworkConfigurationPolicy - name: node-0 + labelSelector: "osp/nncm-config-type=standard" patch: |- - op: remove path: /spec/nodeSelector/node-role.kubernetes.io~1worker - # Optional: add storagemgmt VLAN interface to the SNO node. - # Uncomment this patch AND node_0.storagemgmt_ip in values.yaml. - # Not required for CephHCI (Ceph uses storage network for OSD traffic). - # - target: - # kind: NodeNetworkConfigurationPolicy - # name: node-0 - # patch: |- - # - op: add - # path: /spec/desiredState/interfaces/- - # value: - # description: storagemgmt vlan host interface - # name: storagemgmt - # state: up - # type: vlan - # mtu: _replaced_ - # ipv4: - # address: - # - ip: _replaced_ - # prefix-length: _replaced_ - # dhcp: false - # enabled: true - # ipv6: - # enabled: false - # vlan: - # base-iface: _replaced_ - # id: _replaced_ -# Uncomment these replacements together with the storagemgmt patch above -# replacements: -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.node_0.storagemgmt_ip -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# name: node-0 -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.ip -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.base_iface -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].vlan.base-iface -# options: -# create: true -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.vlan -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].vlan.id -# options: -# create: true -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.mtu -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].mtu -# options: -# create: true -# - source: -# kind: ConfigMap -# name: network-values -# fieldPath: data.storagemgmt.prefix-length -# targets: -# - select: -# kind: NodeNetworkConfigurationPolicy -# fieldPaths: -# - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.prefix-length -# options: -# create: true + # Add external VLAN interface on ospbr for OVN provider network access. + - target: + kind: NodeNetworkConfigurationPolicy + labelSelector: "osp/nncm-config-type=standard" + patch: |- + - op: add + path: /spec/desiredState/interfaces/- + value: + description: external vlan interface on ospbr + name: external + state: up + type: vlan + mtu: _replaced_ + ipv4: + address: + - ip: _replaced_ + prefix-length: _replaced_ + dhcp: false + enabled: true + ipv6: + enabled: false + vlan: + base-iface: _replaced_ + id: _replaced_ + +replacements: + - source: + kind: ConfigMap + name: network-values + fieldPath: data.node_0.external_ip + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].ipv4.address.0.ip + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.base_iface + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].vlan.base-iface + options: + create: true + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.vlan + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].vlan.id + options: + create: true + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.mtu + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].mtu + options: + create: true + - source: + kind: ConfigMap + name: network-values + fieldPath: data.external.prefix-length + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=external].ipv4.address.0.prefix-length + options: + create: true diff --git a/examples/dt/nova/nova05epsilon/README.md b/examples/dt/nova/nova05epsilon/README.md index 645557a36..371491e71 100644 --- a/examples/dt/nova/nova05epsilon/README.md +++ b/examples/dt/nova/nova05epsilon/README.md @@ -28,7 +28,20 @@ This is a collection of CR templates that represent a Red Hat OpenStack Services 5. Between stages 3 and 4, _it is assumed that the user installs Ceph on the compute nodes._ OpenStack K8S CRDs do not provide a way to install Ceph via any sort of combination of CRs. -6. For CI automation, this DT uses `automation/vars/nova05epsilon.yaml` which maps the manual stages above to 9 granular automation steps (NNCP, networking, control-plane, DNS, baremetalhosts, pre-ceph nodeset, pre-ceph deployment, post-ceph nodeset, post-ceph deployment). +6. In stage 5 the `control-plane-post-ceph` kustomization needs the same network values used in stage 2 to preserve endpoint IPs, service types, and DNS configuration. For manual deployment, copy your environment-customized `control-plane/networking/nncp/values.yaml` into `control-plane-post-ceph/network-values.yaml` and populate `control-plane-post-ceph/values.yaml` with base64-encoded Ceph keyring and config. In CI, `ci_gen_kustomize_values` generates `network-values.yaml` in-place using the common Jinja2 template with environment overlays. See [control-plane-post-ceph.md](control-plane-post-ceph.md) for details. + +7. On SNO with a single EDPM compute (single-host CephHCI), the Ceph ingress service (haproxy/keepalived) is not deployed. The default Swift endpoint (`:8080`) is unreachable because no ingress fronts the RGW daemon. Instead, clients must reach RGW directly on the compute's storage IP at port 8082 (the `rgw_frontend_port` set in the Ceph RGW spec). + For CI automation, set `cifmw_cephadm_rgw_port: 8082` and `cifmw_cephadm_rgw_vip: ` in the scenario vars so that `cifmw_cephadm` creates the Keystone endpoint with the correct address. + For manual deployment, after installing Ceph, update the Swift endpoints in Keystone to point at the RGW daemon directly: + + ```shell + STORAGE_IP= + for ep_id in $(openstack endpoint list --service object-store -f value -c ID); do + openstack endpoint set --url "http://${STORAGE_IP}:8082/swift/v1/AUTH_%(tenant_id)s" "$ep_id" + done + ``` + +8. For CI automation, this DT uses `automation/vars/nova05epsilon.yaml` which maps the manual stages above to 10 granular automation steps (NNCP, networking, control-plane, DNS, baremetalhosts, pre-ceph nodeset, pre-ceph deployment, control-plane-post-ceph, post-ceph nodeset, post-ceph deployment). ## Host Configuration @@ -57,7 +70,9 @@ All stages must be executed in the order listed below. Everything is required un 1. [Install the OpenStack K8S operators and their dependencies](../../../common/) 2. [Configuring networking and deploy the OpenStack control plane](control-plane.md) 3. [Configure and deploy the initial data plane to prepare for Ceph installation](dataplane-pre-ceph.md) -4. [Update the control plane and finish deploying the data plane after Ceph has been installed](dataplane-post-ceph.md) +4. Install Ceph on the compute nodes (without changing OpenStack CP CR) +5. [Update the control plane with Ceph backend configuration](control-plane-post-ceph.md) +6. [Finish deploying the data plane after Ceph has been installed](dataplane-post-ceph.md) ## Extending to a Full DCN Deployment diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph.md b/examples/dt/nova/nova05epsilon/control-plane-post-ceph.md new file mode 100644 index 000000000..7a5743e02 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph.md @@ -0,0 +1,79 @@ +# Update the control plane with Ceph backend configuration + +## Assumptions + +- The [pre-ceph data plane](dataplane-pre-ceph.md) has been deployed +- Ceph has been installed on the compute nodes +- The `ceph-conf-files` secret will be created during this step + +## Initialize + +Switch to the "openstack" namespace + +```shell +oc project openstack +``` + +Change to the control-plane-post-ceph directory + +```shell +cd architecture/examples/dt/nova/nova05epsilon/control-plane-post-ceph +``` + +## Prepare values files + +The kustomization requires three values files: + +**network-values.yaml** — Provides storageClass, bridgeName, endpoint +annotations, and DNS options that the lib components need. This file +must match the `network-values` used in the initial control-plane stage +to preserve all endpoint IPs, service types, and DNS configuration. +In CI, `ci_gen_kustomize_values` generates it from the automation vars +(`src_file: network-values.yaml`). For manual deployment, copy the +environment-customized file from the control-plane networking stage: + +```shell +cp ../control-plane/networking/nncp/values.yaml network-values.yaml +``` + +Edit `network-values.yaml` and replace any remaining `CHANGEME` +placeholders to match your environment (same values used in stage 2). + +**values.yaml** — Ceph configuration for the `ceph-conf-files` secret. +Replace the `CHANGEME` placeholders with base64-encoded Ceph keyring +and config from your Ceph deployment: + +```shell +vi values.yaml +# Set data.ceph_conf."ceph.client.openstack.keyring" to: +# base64 -w0 /etc/ceph/ceph.client.openstack.keyring +# Set data.ceph_conf."ceph.conf" to: +# base64 -w0 /etc/ceph/ceph.conf +``` + +**service-values.yaml** — Edit if you need to adjust the Glance RBD +backend or Ceph extraMounts configuration: + +```shell +vi service-values.yaml +``` + +## Update the control plane + +Generate the control-plane-post-ceph CRs: + +```shell +kustomize build > control-plane-post-ceph.yaml +``` + +Apply the CRs: + +```shell +oc apply -f control-plane-post-ceph.yaml +``` + +Wait for the control plane to be ready: + +```shell +oc wait osctlplane controlplane --for condition=Ready --timeout=1200s +``` diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml new file mode 100644 index 000000000..432094a6f --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../network-values + - ../../../../../dt/nova/nova05epsilon/control-plane-post-ceph + +resources: + - values.yaml + - service-values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml new file mode 100644 index 000000000..e74585df4 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/service-values.yaml @@ -0,0 +1,101 @@ +# local-config: referenced, but not emitted by kustomize +# +# Post-Ceph control plane update: adds Glance RBD backend, Ceph +# extraMounts, and preserves all fields from the initial control-plane +# stage that lib/control-plane/base would otherwise reset to defaults. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-values + annotations: + config.kubernetes.io/local-config: "true" +data: + preserveJobs: true + notificationsBus: + cluster: rabbitmq + tls: + caBundleSecretName: custom-ca-certs + podLevel: + enabled: true + neutron: + customServiceConfig: | + [ml2] + mechanism_drivers = ovn + nova: + apiServiceTemplate: + customServiceConfig: | + [pci] + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + [filter_scheduler] + pci_in_placement = True + cell0: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + cell1: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + telemetry: + enabled: true + template: + ceilometer: + enabled: true + logging: + enabled: false + metricStorage: + dashboardsEnabled: true + enabled: true + dataplaneNetwork: ctlplane + networkAttachments: + - ctlplane + monitoringStack: + alertingEnabled: true + alertmanagerConfig: + replicas: 1 + scrapeInterval: 30s + storage: + persistent: + pvcStorageClass: lvms-local-storage + pvcStorageRequest: 20G + pvcStorageSelector: {} + retention: 24h + strategy: persistent + swift: + enabled: false + glance: + default: + replicas: 1 + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:rbd + [glance_store] + default_backend = default_backend + [default_backend] + rbd_store_ceph_conf = /etc/ceph/ceph.conf + store_description = "Ceph RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + extraMounts: + - name: v1 + region: r1 + extraVol: + - propagation: + - GlanceAPI + extraVolType: Ceph + volumes: + - name: ceph + secret: + secretName: ceph-conf-files + mounts: + - name: ceph + mountPath: /etc/ceph + readOnly: true diff --git a/examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml new file mode 100644 index 000000000..c8bdfc336 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/control-plane-post-ceph/values.yaml @@ -0,0 +1,18 @@ +# local-config: referenced, but not emitted by kustomize +# +# Ceph config for the control-plane-post-ceph stage. +# In CI, ci_gen_kustomize_values populates ceph_conf from /tmp/ files +# written by the ceph.yml hook (via the edpm-nodeset-values-post-ceph +# Jinja2 template). For manual use, replace CHANGEME values with +# base64-encoded Ceph keyring and conf. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-nodeset-values-post-ceph + annotations: + config.kubernetes.io/local-config: "true" +data: + ceph_conf: + ceph.client.openstack.keyring: CHANGEME_CEPH_KEYRING + ceph.conf: CHANGEME_CEPH_CONF diff --git a/examples/dt/nova/nova05epsilon/control-plane.md b/examples/dt/nova/nova05epsilon/control-plane.md index 3697b0f7f..3f54955d7 100644 --- a/examples/dt/nova/nova05epsilon/control-plane.md +++ b/examples/dt/nova/nova05epsilon/control-plane.md @@ -18,15 +18,15 @@ Change to the nova05epsilon directory cd architecture/examples/dt/nova/nova05epsilon/control-plane ``` -Edit the [networking/nncp/values.yaml](control-plane/networking/nncp/values.yaml) -and [networking/dns/values.yaml](control-plane/networking/dns/values.yaml) -files to suit your environment. Service values (Glance, Nova PCI, telemetry) -are configured in the top-level [service-values.yaml](service-values.yaml) -and applied during the post-ceph stage rebuild. +Edit the [networking/nncp/values.yaml](control-plane/networking/nncp/values.yaml), +[networking/dns/values.yaml](control-plane/networking/dns/values.yaml) and +[service-values.yaml](control-plane/service-values.yaml) files to suit +your environment. ```shell vi networking/nncp/values.yaml vi networking/dns/values.yaml +vi service-values.yaml ``` ## Apply node network configuration @@ -113,7 +113,11 @@ oc -n openshift-dns wait dns.operator/default --for condition=Available --timeou ``` Now wait for the full control plane readiness before proceeding to -the data plane stages: +the data plane stages. Note: the initial control plane deploys +without Ceph configuration (Glance uses defaults). Ceph backend +and extraMounts are added in the +[control-plane-post-ceph](control-plane-post-ceph.md) stage after +Ceph is installed. ```shell oc wait osctlplane controlplane --for condition=Ready --timeout=600s diff --git a/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml index 127d1b018..81be4a0fc 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/kustomization.yaml @@ -3,8 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization components: + - ../network-values - ../../../../../dt/nova/nova05epsilon/ resources: - - networking/nncp/values.yaml - service-values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml index aabb8ba24..ab672f640 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/networking/kustomization.yaml @@ -3,7 +3,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization components: + - ../../network-values - ../../../../../../dt/nova/nova05epsilon/networking - -resources: - - nncp/values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml index 4006234f5..6d0b2d85e 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/kustomization.yaml @@ -18,12 +18,9 @@ transformers: create: true components: + - ../../../network-values - ../../../../../../../dt/nova/nova05epsilon/networking/nncp # To expose storagemgmt VLAN on the SNO node, uncomment the patch and # replacements block in dt/nova/nova05epsilon/networking/nncp/kustomization.yaml # and uncomment node_0.storagemgmt_ip in values.yaml. # Not required for CephHCI (Ceph uses the storage network for OSD traffic). - - -resources: - - values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml b/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml index c87917e6b..107e3bba9 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml +++ b/examples/dt/nova/nova05epsilon/control-plane/service-values.yaml @@ -14,3 +14,67 @@ data: cluster: rabbitmq tls: caBundleSecretName: "" + + # --- Below are VA/DT-specific data --- + neutron: + customServiceConfig: | + [ml2] + mechanism_drivers = ovn + ovn: + ovnController: + nicMappings: + datacentre: ospbr + glance: + default: + replicas: 0 + swift: + enabled: false + telemetry: + enabled: true + template: + ceilometer: + enabled: true + logging: + enabled: false + metricStorage: + dashboardsEnabled: true + enabled: true + dataplaneNetwork: ctlplane + networkAttachments: + - ctlplane + monitoringStack: + alertingEnabled: true + alertmanagerConfig: + replicas: 1 + scrapeInterval: 30s + storage: + persistent: + pvcStorageClass: lvms-local-storage + pvcStorageRequest: 20G + pvcStorageSelector: {} + retention: 24h + strategy: persistent + # Nova GPU PCI passthrough configuration + nova: + apiServiceTemplate: + customServiceConfig: | + [pci] + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + [filter_scheduler] + pci_in_placement = True + cell0: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + cell1: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + # extraMounts for Ceph are applied during the post-ceph stage + # (see edpm-post-ceph/ kustomization) after ceph-conf-files secret exists. diff --git a/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md b/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md index 130215b33..c1f917424 100644 --- a/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md +++ b/examples/dt/nova/nova05epsilon/dataplane-post-ceph.md @@ -1,9 +1,10 @@ -# Update the control plane and finish deploying the data plane after Ceph has been installed +# Finish deploying the data plane after Ceph has been installed ## Assumptions - The [pre-ceph data plane](dataplane-pre-ceph.md) has been deployed - Ceph has been installed on the compute nodes +- The [control plane has been updated with Ceph configuration](control-plane-post-ceph.md) ## Initialize @@ -19,9 +20,8 @@ Change to the nova05epsilon directory cd architecture/examples/dt/nova/nova05epsilon ``` -Edit the [values.yaml](values.yaml) and [service-values.yaml](service-values.yaml) -files to suit your environment. In particular, update the Ceph configuration -placeholders in `values.yaml`: +Edit the [values.yaml](values.yaml) file to suit your environment. +In particular, update the Ceph configuration placeholders: - **`data.ceph_conf`** (DCN convention): A dict mapping Ceph filenames to base64-encoded content. For a single-site deployment, use plain filenames. @@ -48,13 +48,11 @@ Replace `` with the Ceph cluster FSID (from `ceph fsid`). ```shell vi values.yaml -vi service-values.yaml ``` -## Update the control plane and deploy the post-ceph dataplane +## Deploy the post-ceph dataplane -Generate the post-ceph CRs (this includes both the updated control plane -and the post-ceph nodeset): +Generate the post-ceph dataplane CRs: ```shell kustomize build > dataplane-nodeset.yaml @@ -66,12 +64,6 @@ Apply the CRs: oc apply -f dataplane-nodeset.yaml ``` -Wait for control plane to be available: - -```shell -oc wait osctlplane controlplane --for condition=Ready --timeout=600s -``` - Wait for the nodeset setup to complete: ```shell diff --git a/examples/dt/nova/nova05epsilon/kustomization.yaml b/examples/dt/nova/nova05epsilon/kustomization.yaml index b4590d7e7..e08a55e7c 100644 --- a/examples/dt/nova/nova05epsilon/kustomization.yaml +++ b/examples/dt/nova/nova05epsilon/kustomization.yaml @@ -4,10 +4,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization components: + - network-values - ../../../../dt/nova/nova05epsilon/edpm-post-ceph/nodeset resources: - - control-plane/networking/nncp/values.yaml - edpm-pre-ceph/nodeset/values.yaml - service-values.yaml - values.yaml diff --git a/examples/dt/nova/nova05epsilon/network-values/kustomization.yaml b/examples/dt/nova/nova05epsilon/network-values/kustomization.yaml new file mode 100644 index 000000000..ae3071458 --- /dev/null +++ b/examples/dt/nova/nova05epsilon/network-values/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - values.yaml diff --git a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml b/examples/dt/nova/nova05epsilon/network-values/values.yaml similarity index 71% rename from examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml rename to examples/dt/nova/nova05epsilon/network-values/values.yaml index e7a8caba2..66b1a66de 100644 --- a/examples/dt/nova/nova05epsilon/control-plane/networking/nncp/values.yaml +++ b/examples/dt/nova/nova05epsilon/network-values/values.yaml @@ -30,7 +30,11 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_CTLPLANE_CIDR", "range_start": "CHANGEME_SNO_CTLPLANE_NAD_START", - "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END" + "range_end": "CHANGEME_SNO_CTLPLANE_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_CTLPLANE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" }, + { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_CTLPLANE_GW" } + ] } }, { "type": "tuning", @@ -50,6 +54,13 @@ data: gateway: CHANGEME_SNO_CTLPLANE_GW name: subnet1 vlan: CHANGEME_SNO_CTLPLANE_VLAN + routes: + - destination: CHANGEME_EDPM_CTLPLANE_CIDR + nexthop: CHANGEME_SNO_CTLPLANE_GW + # Pods with only ctlplane attached (e.g. tempest) need this + # to reach Ceph RGW on the remote storage network. + - destination: CHANGEME_EDPM_STORAGE_CIDR + nexthop: CHANGEME_SNO_CTLPLANE_GW # CHANGEME: site4 EDPM computes -- replace CIDRs/gateway/VLAN - allocationRanges: - end: CHANGEME_EDPM_CTLPLANE_END @@ -58,6 +69,9 @@ data: gateway: CHANGEME_EDPM_CTLPLANE_GW name: subnet2 vlan: CHANGEME_EDPM_CTLPLANE_VLAN + routes: + - destination: CHANGEME_SNO_CTLPLANE_CIDR + nexthop: CHANGEME_EDPM_CTLPLANE_GW datacentre: net-attach-def: | { @@ -84,8 +98,31 @@ data: values: - CHANGEME_SNO_DNS_SERVER external: + base_iface: ospbr dnsDomain: external.example.com + iface: external mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.4.0", + "name": "external", + "plugins": [{ + "type": "macvlan", + "master": "external", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "CHANGEME_SNO_EXTERNAL_CIDR", + "range_start": "CHANGEME_SNO_EXTERNAL_NAD_START", + "range_end": "CHANGEME_SNO_EXTERNAL_NAD_END" + } + }, { + "type": "tuning", + "sysctl": { + "net.ipv6.conf.IFNAME.accept_ra": "0" + } + }] + } prefix-length: 24 subnets: - allocationRanges: @@ -126,7 +163,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_INTAPI_CIDR", "range_start": "CHANGEME_SNO_INTAPI_NAD_START", - "range_end": "CHANGEME_SNO_INTAPI_NAD_END" + "range_end": "CHANGEME_SNO_INTAPI_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_INTAPI_CIDR", "gw": "CHANGEME_SNO_INTAPI_GW" } + ] } }, { "type": "tuning", @@ -141,20 +181,29 @@ data: - end: CHANGEME_SNO_INTAPI_END start: CHANGEME_SNO_INTAPI_START cidr: CHANGEME_SNO_INTAPI_CIDR + gateway: CHANGEME_SNO_INTAPI_GW name: subnet1 vlan: CHANGEME_SNO_INTAPI_VLAN + routes: + - destination: CHANGEME_EDPM_INTAPI_CIDR + nexthop: CHANGEME_SNO_INTAPI_GW # CHANGEME: site4 internalapi subnet - allocationRanges: - end: CHANGEME_EDPM_INTAPI_END start: CHANGEME_EDPM_INTAPI_START cidr: CHANGEME_EDPM_INTAPI_CIDR + gateway: CHANGEME_EDPM_INTAPI_GW name: subnet2 vlan: CHANGEME_EDPM_INTAPI_VLAN + routes: + - destination: CHANGEME_SNO_INTAPI_CIDR + nexthop: CHANGEME_EDPM_INTAPI_GW vlan: CHANGEME_SNO_INTAPI_VLAN lbServiceType: LoadBalancer # SNO: single OCP node only node_0: ctlplane_ip: CHANGEME_SNO_CTLPLANE_IP + external_ip: CHANGEME_SNO_EXTERNAL_IP internalapi_ip: CHANGEME_SNO_INTAPI_IP name: CHANGEME_SNO_NODE_NAME storage_ip: CHANGEME_SNO_STORAGE_IP @@ -183,7 +232,24 @@ data: metallb.universe.tf/address-pool: internalapi metallb.universe.tf/loadBalancerIPs: CHANGEME_SNO_RABBITMQ_CELL1_IP routes: - config: [] + config: + # CHANGEME: host-level routes for cross-site reachability (EDPM subnet2) + - destination: CHANGEME_EDPM_CTLPLANE_CIDR + next-hop-address: CHANGEME_SNO_CTLPLANE_GW + next-hop-interface: ospbr + table-id: 254 + - destination: CHANGEME_EDPM_INTAPI_CIDR + next-hop-address: CHANGEME_SNO_INTAPI_GW + next-hop-interface: internalapi + table-id: 254 + - destination: CHANGEME_EDPM_STORAGE_CIDR + next-hop-address: CHANGEME_SNO_STORAGE_GW + next-hop-interface: storage + table-id: 254 + - destination: CHANGEME_EDPM_TENANT_CIDR + next-hop-address: CHANGEME_SNO_TENANT_GW + next-hop-interface: tenant + table-id: 254 storage: base_iface: CHANGEME_SNO_IFACE dnsDomain: storage.example.com @@ -206,7 +272,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_STORAGE_CIDR", "range_start": "CHANGEME_SNO_STORAGE_NAD_START", - "range_end": "CHANGEME_SNO_STORAGE_NAD_END" + "range_end": "CHANGEME_SNO_STORAGE_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_STORAGE_CIDR", "gw": "CHANGEME_SNO_STORAGE_GW" } + ] } }, { "type": "tuning", @@ -221,15 +290,23 @@ data: - end: CHANGEME_SNO_STORAGE_END start: CHANGEME_SNO_STORAGE_START cidr: CHANGEME_SNO_STORAGE_CIDR + gateway: CHANGEME_SNO_STORAGE_GW name: subnet1 vlan: CHANGEME_SNO_STORAGE_VLAN + routes: + - destination: CHANGEME_EDPM_STORAGE_CIDR + nexthop: CHANGEME_SNO_STORAGE_GW # CHANGEME: site4 storage subnet - allocationRanges: - end: CHANGEME_EDPM_STORAGE_END start: CHANGEME_EDPM_STORAGE_START cidr: CHANGEME_EDPM_STORAGE_CIDR + gateway: CHANGEME_EDPM_STORAGE_GW name: subnet2 vlan: CHANGEME_EDPM_STORAGE_VLAN + routes: + - destination: CHANGEME_SNO_STORAGE_CIDR + nexthop: CHANGEME_EDPM_STORAGE_GW vlan: CHANGEME_SNO_STORAGE_VLAN storageClass: lvms-local-storage storagemgmt: @@ -250,7 +327,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_STGMGMT_CIDR", "range_start": "CHANGEME_SNO_STGMGMT_NAD_START", - "range_end": "CHANGEME_SNO_STGMGMT_NAD_END" + "range_end": "CHANGEME_SNO_STGMGMT_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_STGMGMT_CIDR", "gw": "CHANGEME_SNO_STGMGMT_GW" } + ] } }, { "type": "tuning", @@ -265,15 +345,23 @@ data: - end: CHANGEME_SNO_STGMGMT_END start: CHANGEME_SNO_STGMGMT_START cidr: CHANGEME_SNO_STGMGMT_CIDR + gateway: CHANGEME_SNO_STGMGMT_GW name: subnet1 vlan: CHANGEME_SNO_STGMGMT_VLAN + routes: + - destination: CHANGEME_EDPM_STGMGMT_CIDR + nexthop: CHANGEME_SNO_STGMGMT_GW # CHANGEME: site4 storagemgmt subnet - allocationRanges: - end: CHANGEME_EDPM_STGMGMT_END start: CHANGEME_EDPM_STGMGMT_START cidr: CHANGEME_EDPM_STGMGMT_CIDR + gateway: CHANGEME_EDPM_STGMGMT_GW name: subnet2 vlan: CHANGEME_EDPM_STGMGMT_VLAN + routes: + - destination: CHANGEME_SNO_STGMGMT_CIDR + nexthop: CHANGEME_EDPM_STGMGMT_GW vlan: CHANGEME_SNO_STGMGMT_VLAN tenant: base_iface: CHANGEME_SNO_IFACE @@ -297,7 +385,10 @@ data: "type": "whereabouts", "range": "CHANGEME_SNO_TENANT_CIDR", "range_start": "CHANGEME_SNO_TENANT_NAD_START", - "range_end": "CHANGEME_SNO_TENANT_NAD_END" + "range_end": "CHANGEME_SNO_TENANT_NAD_END", + "routes": [ + { "dst": "CHANGEME_EDPM_TENANT_CIDR", "gw": "CHANGEME_SNO_TENANT_GW" } + ] } }, { "type": "tuning", @@ -312,15 +403,23 @@ data: - end: CHANGEME_SNO_TENANT_END start: CHANGEME_SNO_TENANT_START cidr: CHANGEME_SNO_TENANT_CIDR + gateway: CHANGEME_SNO_TENANT_GW name: subnet1 vlan: CHANGEME_SNO_TENANT_VLAN + routes: + - destination: CHANGEME_EDPM_TENANT_CIDR + nexthop: CHANGEME_SNO_TENANT_GW # CHANGEME: site4 tenant subnet - allocationRanges: - end: CHANGEME_EDPM_TENANT_END start: CHANGEME_EDPM_TENANT_START cidr: CHANGEME_EDPM_TENANT_CIDR + gateway: CHANGEME_EDPM_TENANT_GW name: subnet2 vlan: CHANGEME_EDPM_TENANT_VLAN + routes: + - destination: CHANGEME_SNO_TENANT_CIDR + nexthop: CHANGEME_EDPM_TENANT_GW vlan: CHANGEME_SNO_TENANT_VLAN kind: ConfigMap diff --git a/examples/dt/nova/nova05epsilon/service-values.yaml b/examples/dt/nova/nova05epsilon/service-values.yaml index 6033bb80c..572766665 100644 --- a/examples/dt/nova/nova05epsilon/service-values.yaml +++ b/examples/dt/nova/nova05epsilon/service-values.yaml @@ -55,10 +55,12 @@ data: - ctlplane monitoringStack: alertingEnabled: true + alertmanagerConfig: + replicas: 1 scrapeInterval: 30s storage: persistent: - pvcStorageClass: local-storage + pvcStorageClass: lvms-local-storage pvcStorageRequest: 20G pvcStorageSelector: {} retention: 24h diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml index 8db00ed37..72a1620d5 100644 --- a/zuul.d/validations.yaml +++ b/zuul.d/validations.yaml @@ -307,8 +307,12 @@ cifmw_architecture_scenario: nova04delta-adoption - job: files: + - automation/mocks/nova05epsilon.yaml + - automation/net-env/nova05epsilon.yaml - examples/dt/nova/nova05epsilon + - examples/dt/nova/nova05epsilon/network-values - examples/dt/nova/nova05epsilon/control-plane + - examples/dt/nova/nova05epsilon/control-plane-post-ceph - examples/dt/nova/nova05epsilon/control-plane/networking - examples/dt/nova/nova05epsilon/control-plane/networking/dns - examples/dt/nova/nova05epsilon/control-plane/networking/nncp @@ -321,6 +325,7 @@ parent: rhoso-architecture-base-job vars: cifmw_architecture_scenario: nova05epsilon + cifmw_networking_env_def_file: automation/net-env/nova05epsilon.yaml - job: files: - examples/va/nvidia-mdev/control-plane