diff --git a/docs_user/assemblies/assembly_adopting-openstack-control-plane-services.adoc b/docs_user/assemblies/assembly_adopting-openstack-control-plane-services.adoc index e17d6a308..134b8cf68 100644 --- a/docs_user/assemblies/assembly_adopting-openstack-control-plane-services.adoc +++ b/docs_user/assemblies/assembly_adopting-openstack-control-plane-services.adoc @@ -34,6 +34,8 @@ include::../modules/proc_adopting-the-compute-service.adoc[leveloffset=+1] include::../modules/proc_adopting-the-block-storage-service.adoc[leveloffset=+1] +include::../modules/proc_adopting-block-storage-service-with-dcn-backend.adoc[leveloffset=+1] + include::../modules/proc_adopting-the-openstack-dashboard.adoc[leveloffset=+1] include::../assemblies/assembly_adopting-the-shared-file-systems-service.adoc[leveloffset=+1] diff --git a/docs_user/assemblies/assembly_adopting-the-data-plane.adoc b/docs_user/assemblies/assembly_adopting-the-data-plane.adoc index 2fd72e8c0..d0ef53c54 100644 --- a/docs_user/assemblies/assembly_adopting-the-data-plane.adoc +++ b/docs_user/assemblies/assembly_adopting-the-data-plane.adoc @@ -26,6 +26,8 @@ include::../modules/proc_adopting-compute-services-to-the-data-plane.adoc[levelo include::../modules/proc_configuring-dcn-data-plane-nodesets.adoc[leveloffset=+1] +include::../modules/proc_adopting-compute-services-with-dcn-backend.adoc[leveloffset=+1] + include::../modules/proc_performing-a-fast-forward-upgrade-on-compute-services.adoc[leveloffset=+1] include::../modules/proc_adopting-networker-services-to-the-data-plane.adoc[leveloffset=+1] diff --git a/docs_user/assemblies/assembly_adopting-the-image-service.adoc b/docs_user/assemblies/assembly_adopting-the-image-service.adoc index 684eb5db5..7f16d97f2 100644 --- a/docs_user/assemblies/assembly_adopting-the-image-service.adoc +++ b/docs_user/assemblies/assembly_adopting-the-image-service.adoc @@ -31,6 +31,8 @@ include::../modules/proc_adopting-image-service-with-nfs-backend.adoc[leveloffse include::../modules/proc_adopting-image-service-with-ceph-backend.adoc[leveloffset=+1] +include::../modules/proc_adopting-image-service-with-dcn-backend.adoc[leveloffset=+1] + include::../modules/proc_verifying-the-image-service-adoption.adoc[leveloffset=+1] ifdef::parent-context[:context: {parent-context}] diff --git a/docs_user/assemblies/assembly_migrating-ceph-cluster.adoc b/docs_user/assemblies/assembly_migrating-ceph-cluster.adoc index 804463443..fa73ccf46 100644 --- a/docs_user/assemblies/assembly_migrating-ceph-cluster.adoc +++ b/docs_user/assemblies/assembly_migrating-ceph-cluster.adoc @@ -31,6 +31,7 @@ already part of the {CephCluster} cluster. Before you begin the migration, complete the tasks in your {rhos_prev_long} {rhos_prev_ver} environment. For more information, see xref:red-hat-ceph-storage-prerequisites_configuring-network[{Ceph} prerequisites]. +include::../modules/con_ceph-migration-dcn.adoc[leveloffset=+1] include::../modules/con_ceph-daemon-cardinality.adoc[leveloffset=+1] diff --git a/docs_user/modules/con_adoption-limitations.adoc b/docs_user/modules/con_adoption-limitations.adoc index 678502a09..38b4e5e2d 100644 --- a/docs_user/modules/con_adoption-limitations.adoc +++ b/docs_user/modules/con_adoption-limitations.adoc @@ -29,7 +29,6 @@ Unsupported features:: + The adoption process does not support the following features: + -* Distributed Compute Node (DCN) architecture with storage services at remote or edge sites * DNS-as-a-service (designate) * {loadbalancer_first_ref} * Adopting Border Gateway Protocol (BGP) environments to the {rhos_acro} data plane diff --git a/docs_user/modules/con_ceph-migration-dcn.adoc b/docs_user/modules/con_ceph-migration-dcn.adoc new file mode 100644 index 000000000..8589b6d9f --- /dev/null +++ b/docs_user/modules/con_ceph-migration-dcn.adoc @@ -0,0 +1,44 @@ +:_mod-docs-content-type: CONCEPT +[id="ceph-migration-dcn_{context}"] + += {Ceph} migration for Distributed Compute Node deployments + +[role="_abstract"] +Before you adopt your Distributed Compute Node (DCN) deployments that host {CephCluster} clusters on Compute nodes at edge sites so that your architecture runs on {rhos_long} (RHOSO), be aware of important considerations. + +Supported edge storage topologies:: DCN deployments support the following storage topologies at edge sites: ++ +* Hyperconverged Infrastructure (HCI): {Ceph} daemons run on Compute nodes at each edge site. +* {OpenStackPreviousInstaller}-deployed dedicated storage: {Ceph} runs on separate storage nodes deployed by {OpenStackPreviousInstaller}. +* External {Ceph} cluster: Edge sites connect to pre-existing {CephCluster} clusters not managed by {OpenStackPreviousInstaller}. + +Central site {Ceph} migration:: For the central site, migrate {Ceph} daemons from the {OpenStackShort} Controller nodes by using the same process as a non-DCN deployment. For more information, see xref:ceph-daemon-cardinality_migrating-ceph[{Ceph} daemon cardinality]. + +Edge site {Ceph} migration:: For edge sites that use HCI or {OpenStackPreviousInstaller}-deployed dedicated storage, the {Ceph} daemons can continue to run on their current nodes without migration. The Compute nodes or dedicated storage nodes at edge sites are not decommissioned during adoption, so the {Ceph} daemons remain operational. ++ +For edge sites that use external {Ceph} clusters, no migration is required because the {CephCluster} cluster is not managed by {OpenStackPreviousInstaller}. + +{Ceph} back-end configuration and key distribution:: In a DCN deployment, each site has its own {CephCluster} cluster with its own configuration file and {Ceph} keyring. These must be stored in Kubernetes secrets and mounted into the appropriate {rhos_long} service pods. ++ +Rather than storing all {Ceph} keys in a single secret accessible to every pod, the recommended approach is to create one secret per site containing only the keys that site actually needs. This limits the security impact if a site is compromised: a pod at an edge site can authenticate only to its local {CephCluster} cluster and the central cluster, not to the {Ceph} keyrings of other edge sites. ++ +The key distribution rule for N sites is: ++ +* The central site (site 0) receives the {Ceph} keys and configuration for all clusters, because central services such as {image_service} use the `split` back end and must be able to copy images to and from any site. +* Each edge site (site 1 through N) receives only the keys for the central cluster and its own local cluster. ++ +For example, in a three-site deployment with a central site and two edge sites: ++ +---- +ceph-conf-central -> central.conf + central.keyring + dcn1.conf + dcn1.keyring + dcn2.conf + dcn2.keyring + +ceph-conf-dcn1 -> central.conf + central.keyring + dcn1.conf + dcn1.keyring + +ceph-conf-dcn2 -> central.conf + central.keyring + dcn2.conf + dcn2.keyring +---- ++ +The per-site secrets are created and then mounted into the appropriate pods using `extraMounts` propagation labels. The procedure in xref:configuring-a-ceph-backend_migrating-databases[Configuring a {Ceph} back end] covers both creating the secrets and applying the propagation labels so that each pod receives only its site-specific keys. diff --git a/docs_user/modules/proc_adopting-block-storage-service-with-dcn-backend.adoc b/docs_user/modules/proc_adopting-block-storage-service-with-dcn-backend.adoc new file mode 100644 index 000000000..11ee5be95 --- /dev/null +++ b/docs_user/modules/proc_adopting-block-storage-service-with-dcn-backend.adoc @@ -0,0 +1,216 @@ +:_mod-docs-content-type: PROCEDURE +[id="adopting-block-storage-service-with-dcn-backend_{context}"] + += Adopt the {block_storage} with multiple {Ceph} back ends (DCN) + +[role="_abstract"] +Adopt the {block_storage_first_ref} in a Distributed Compute Node (DCN) deployment where multiple {CephCluster} clusters provide storage at different sites. You can deploy multiple `CinderVolume` instances, one for each availability zone, with each volume service configured to use its local {Ceph} cluster. + +During adoption, the {block_storage} volume services that ran on edge site Compute nodes are migrated to run on {rhocp_long} at the central site. Although the control path for API requests now traverses the WAN to reach the {block_storage} running on {rhocp_long}, the data path remains local. Volume data continues to be stored in the {Ceph} cluster at each edge site. When you create a volume or clone a volume from a snapshot, the operation occurs entirely within the local {Ceph} cluster. This preserves data locality. + +.Prerequisites + +* You have completed the previous adoption steps. +* The per-site {Ceph} secrets (`ceph-conf-central`, `ceph-conf-dcn1`, `ceph-conf-dcn2`) exist and contain the configuration and keyrings for each site's {Ceph} cluster. For more information, see xref:configuring-a-ceph-backend_migrating-databases[Configuring a {Ceph} back end]. +* The `extraMounts` property of the `OpenStackControlPlane` custom resource (CR) is configured to mount the {Ceph} configuration to all {block_storage} instances. +* You have stopped the {block_storage} on all DCN nodes. For more information, see xref:stopping-openstack-services_{context}[Stopping {rhos_prev_long} services]. On edge sites, the {block_storage} volume service runs on Compute nodes with the service name `tripleo_cinder_volume.service`. + +.Procedure + +. Retrieve the `fsid` for each {Ceph} cluster in your DCN deployment. The `fsid` is used as the `rbd_secret_uuid` for libvirt integration: ++ +---- +$ oc get secret ceph-conf-central -o json | jq -r '.data | to_entries[] | select(.key | endswith(".conf")) | "\(.key): \(.value | @base64d)"' | grep fsid +---- + +. Create a patch file for the {block_storage} with multiple {Ceph} back ends. The following example shows a DCN deployment with a central site and two edge sites: ++ +[subs="+quotes"] +---- +$ cat << EOF > cinder_dcn_patch.yaml +spec: + cinder: + enabled: true + template: + cinderAPI: + customServiceConfig: | + [DEFAULT] + default_availability_zone = az-central + cinderScheduler: + replicas: 1 + cinderVolumes: + central: + networkAttachments: + - storage + replicas: 1 + customServiceConfig: | + [DEFAULT] + enabled_backends = central + glance_api_servers = http://glance-central-internal.openstack.svc:9292 + [central] + backend_host = hostgroup + volume_backend_name = central + volume_driver = cinder.volume.drivers.rbd.RBDDriver + rbd_ceph_conf = /etc/ceph/central.conf + rbd_user = openstack + rbd_pool = volumes + rbd_flatten_volume_from_snapshot = False + report_discard_supported = True + rbd_secret_uuid = ** + rbd_cluster_name = central + backend_availability_zone = az-central + dcn1: + networkAttachments: + - storage + replicas: 1 + customServiceConfig: | + [DEFAULT] + enabled_backends = dcn1 + glance_api_servers = http://glance-dcn1-internal.openstack.svc:9292 + [dcn1] + backend_host = hostgroup + volume_backend_name = dcn1 + volume_driver = cinder.volume.drivers.rbd.RBDDriver + rbd_ceph_conf = /etc/ceph/dcn1.conf + rbd_user = openstack + rbd_pool = volumes + rbd_flatten_volume_from_snapshot = False + report_discard_supported = True + rbd_secret_uuid = ** + rbd_cluster_name = dcn1 + backend_availability_zone = az-dcn1 + dcn2: + networkAttachments: + - storage + replicas: 1 + customServiceConfig: | + [DEFAULT] + enabled_backends = dcn2 + glance_api_servers = http://glance-dcn2-internal.openstack.svc:9292 + [dcn2] + backend_host = hostgroup + volume_backend_name = dcn2 + volume_driver = cinder.volume.drivers.rbd.RBDDriver + rbd_ceph_conf = /etc/ceph/dcn2.conf + rbd_user = openstack + rbd_pool = volumes + rbd_flatten_volume_from_snapshot = False + report_discard_supported = True + rbd_secret_uuid = ** + rbd_cluster_name = dcn2 + backend_availability_zone = az-dcn2 +EOF +---- ++ +where: + +``:: +Specifies the `fsid` of the central {Ceph} cluster, used as the libvirt secret UUID. + +``:: +Specifies the `fsid` of the DCN1 edge {Ceph} cluster. + +``:: +Specifies the `fsid` of the DCN2 edge {Ceph} cluster. + ++ +[NOTE] +==== +* You must configure each `CinderVolume` with the `backend_availability_zone` value that matches your {compute_service} availability zone for that site, because `cross_az_attach = False` is set in the {compute_service} configuration. If the names do not match, instances cannot attach volumes. Replace the examples (`az-central`, `az-dcn1`, `az-dcn2`) with the names used in your {rhos_prev_long} deployment. +* Each `CinderVolume` points to its local {image_service} API endpoint through `glance_api_servers`. This ensures that volume creation from images uses the local {image_service} and {Ceph} cluster. The examples use `http://` for the {image_service} endpoints. If your {rhos_prev_long} deployment uses TLS for internal endpoints, use `https://` instead, and ensure that you have completed the TLS migration. For more information, see xref:migrating-tls-everywhere_{context}[Migrating TLS-e to the RHOSO deployment]. +* The `rbd_cluster_name` setting identifies which {Ceph} cluster configuration to use from the mounted secrets. +* Adjust the number of edge sites and their names to match your DCN deployment. +==== + +. Patch the `OpenStackControlPlane` CR to deploy the {block_storage} with multiple {Ceph} back ends: ++ +---- +$ oc patch openstackcontrolplane openstack --type=merge --patch-file cinder_dcn_patch.yaml +---- + +. Configure the {block_storage} backup service. In this example, the backup service runs at the central site and uses the central {Ceph} cluster. Add the `cinderBackups` section to your patch file and re-apply it: ++ +[subs="+quotes"] +---- +$ cat << EOF >> cinder_dcn_patch.yaml + cinderBackups: + central: + networkAttachments: + - storage + replicas: 1 + customServiceConfig: | + [DEFAULT] + backup_driver=cinder.backup.drivers.ceph.CephBackupDriver + backup_ceph_conf=/etc/ceph/central.conf + backup_ceph_user=openstack + backup_ceph_pool=backups + storage_availability_zone=az-central +EOF +$ oc patch openstackcontrolplane openstack --type=merge --patch-file cinder_dcn_patch.yaml +---- ++ +[NOTE] +==== +Unlike a single-site {Ceph} deployment where the backup config references `/etc/ceph/ceph.conf`, in a DCN deployment the {Ceph} configuration files in the `ceph-conf-files` secret are named by cluster. Set `backup_ceph_conf` to the path of the {Ceph} configuration file for whichever cluster hosts your `backups` pool. In this example the file is named `central.conf`, so the path is `/etc/ceph/central.conf`. Using a path that does not match a file in the secret will cause the backup service to fail with a `conf_read_file` error. + +Set `storage_availability_zone` to match the availability zone of the volumes you want to back up. The backup scheduler uses this to route backup requests to a service in the correct zone. If the backup service zone does not match the volume zone, backup creation fails with `Service not found for creating backup`. +==== + +. Verify that the {block_storage} volume services are running for each availability zone: ++ +---- +$ openstack volume service list --service cinder-volume + ++------------------+---------------------+------------+---------+-------+----------------------------+ +| Binary | Host | Zone | Status | State | Updated At | ++------------------+---------------------+------------+---------+-------+----------------------------+ +| cinder-volume | hostgroup@central | az-central | enabled | up | 2024-01-01T00:00:00.000000 | +| cinder-volume | hostgroup@dcn1 | az-dcn1 | enabled | up | 2024-01-01T00:00:00.000000 | +| cinder-volume | hostgroup@dcn2 | az-dcn2 | enabled | up | 2024-01-01T00:00:00.000000 | ++------------------+---------------------+------------+---------+-------+----------------------------+ +---- + +. Verify that the {block_storage} backup service is running and in the correct availability zone: ++ +---- +$ openstack volume service list --service cinder-backup + ++---------------+-------------------------+------------+---------+-------+----------------------------+ +| Binary | Host | Zone | Status | State | Updated At | ++---------------+-------------------------+------------+---------+-------+----------------------------+ +| cinder-backup | cinder-backup-central-0 | az-central | enabled | up | 2024-01-01T00:00:00.000000 | ++---------------+-------------------------+------------+---------+-------+----------------------------+ +---- + +. Test the backup service by creating a volume, backing it up, and restoring the backup: ++ +---- +$ openstack volume create --size 1 backup-test-vol + +$ openstack volume backup create --name backup-test-backup backup-test-vol + +$ openstack volume backup show backup-test-backup ++-----------------------+--------------------------------------+ +| Field | Value | ++-----------------------+--------------------------------------+ +| container | backups | +| fail_reason | None | +| name | backup-test-backup | +| size | 1 | +| status | available | ++-----------------------+--------------------------------------+ + +$ openstack volume backup restore backup-test-backup backup-test-restore +---- ++ +[NOTE] +==== +Some versions of the {rhocp_long} client display a `cannot unpack non-iterable VolumeBackupsRestore object` error after the restore command. This is a known issue in the client, the restore operation might not have failed. Verify by checking the restored volume status directly. +==== ++ +---- +$ openstack volume show backup-test-restore -c status -c availability_zone -c os-vol-host-attr:host -f value +available +az-central +hostgroup@central#central +---- diff --git a/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc b/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc index 5406fe19d..024dfff7d 100644 --- a/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc +++ b/docs_user/modules/proc_adopting-compute-services-to-the-data-plane.adoc @@ -264,6 +264,12 @@ EOF [NOTE] For {Ceph} environments with multi-cell configurations, you must name configuration maps and {rhos_prev_long} data plane services similar to the following examples: `nova-custom-ceph-cellX` and `nova-compute-extraconfig-cellX`. ++ +[NOTE] +==== +For Distributed Compute Node (DCN) deployments, do not use the single `nova-cells-global-config` ConfigMap. Create a per-site `ConfigMap` and per-site `OpenStackDataPlaneService` for each site in your DCN deployment. Each site's {compute_service} nodes require a different {Ceph} configuration and a different {image_service} endpoint. For more information, see xref:adopting-compute-services-with-dcn-backend_{context}[Adopting Compute services with multiple Ceph back ends (DCN)]. +==== + . Create the data plane services for {compute_service} cells to enable pre-upgrade workarounds, and to configure the Compute services for your chosen storage back end: + [subs="+quotes"] diff --git a/docs_user/modules/proc_adopting-compute-services-with-dcn-backend.adoc b/docs_user/modules/proc_adopting-compute-services-with-dcn-backend.adoc new file mode 100644 index 000000000..a80af3391 --- /dev/null +++ b/docs_user/modules/proc_adopting-compute-services-with-dcn-backend.adoc @@ -0,0 +1,320 @@ +:_mod-docs-content-type: PROCEDURE +[id="adopting-compute-services-with-dcn-backend_{context}"] + += Adopting {compute_service} services with multiple {Ceph} back ends (DCN) + +[role="_abstract"] +In a Distributed Compute Node (DCN) deployment where {image_service_first_ref} and {block_storage_first_ref} services run on edge Compute nodes, each site has its own {CephCluster} cluster. The {compute_service_first_ref} nodes at each site must be configured with the {Ceph} connection details and {image_service} endpoint for their local site. Because the {image_service} has a separate API endpoint at each site, each site's `OpenStackDataPlaneNodeSet` custom resource (CR) must use a different `OpenStackDataPlaneService` CR that points to the correct {image_service}. + +In a DCN deployment, all node sets belong to a single {compute_service} cell. The central site and each edge site are separate `OpenStackDataPlaneNodeSet` resources within that cell. The per-site `OpenStackDataPlaneService` resources deliver different {Ceph} and {image_service} configurations to each node set while sharing the same cell-level {compute_service} configuration. + +.Prerequisites + +* You have adopted the {image_service} with multiple {Ceph} back ends. For more information, see xref:adopting-image-service-with-dcn-backend_image-service[Adopting the Image service with multiple Ceph back ends]. +* You have adopted the {block_storage} with multiple {Ceph} back ends. For more information, see xref:adopting-block-storage-service-with-dcn-backend_hsm-integration[Adopting the Block Storage service with multiple Ceph back ends]. +* The per-site {Ceph} secrets (`ceph-conf-central`, `ceph-conf-dcn1`, `ceph-conf-dcn2`) exist. For more information, see xref:configuring-a-ceph-backend_migrating-databases[Configuring a {Ceph} back end]. +* Retrieve the `fsid` for each {Ceph} cluster: ++ +---- +$ oc get secret ceph-conf-central -o json | jq -r '.data | to_entries[] | select(.key | endswith(".conf")) | "\(.key): \(.value | @base64d)"' | grep fsid +---- + +.Procedure + +. Set the cell name variable. In a DCN deployment, all node sets belong to a single cell: ++ +---- +$ DEFAULT_CELL_NAME="cell1" +---- + +. Retrieve the `fsid` for each {Ceph} cluster and store them in shell variables: ++ +[subs="+quotes"] +---- +$ CEPH_FSID_CENTRAL=$(oc get secret ceph-conf-central -o json | jq -r '.data.""' | base64 -d | awk '/fsid/{print $3}') +$ CEPH_FSID_DCN1=$(oc get secret ceph-conf-dcn1 -o json | jq -r '.data.""' | base64 -d | awk '/fsid/{print $3}') +$ CEPH_FSID_DCN2=$(oc get secret ceph-conf-dcn2 -o json | jq -r '.data.""' | base64 -d | awk '/fsid/{print $3}') +---- ++ +where: + +``:: +Specifies the name of the {Ceph} configuration file for the central site in the `ceph-conf-central` secret. + +``:: +Specifies the name of the {Ceph} configuration file for an edge site in the `ceph-conf-dcn1` secret. + +``:: +Specifies the name of the {Ceph} configuration file for an additional edge site in the `ceph-conf-dcn2` secret. + +. Create a `ConfigMap` for each site. Each `ConfigMap` contains the {Ceph} and {image_service} configuration specific to that site. ++ +The following example creates `ConfigMap` resources for a central site and two edge sites. ++ +.. Create the `ConfigMap` for the central site: ++ +---- +$ oc apply -f - < glance_dcn_patch.yaml +spec: + glance: + enabled: true + template: + databaseInstance: openstack + databaseAccount: glance + keystoneEndpoint: central + storage: + storageRequest: *<10G>* + glanceAPIs: + central: + type: split + replicas: 3 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + *metallb.universe.tf/loadBalancerIPs: <172.17.0.80>* + spec: + type: LoadBalancer + networkAttachments: + - storage + customServiceConfig: | + [DEFAULT] + enabled_import_methods = [web-download,copy-image,glance-direct] + enabled_backends = central:rbd,dcn1:rbd,dcn2:rbd + [glance_store] + default_backend = central + [central] + rbd_store_ceph_conf = /etc/ceph/central.conf + store_description = "Central RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + [dcn1] + rbd_store_ceph_conf = /etc/ceph/dcn1.conf + store_description = "DCN1 RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + [dcn2] + rbd_store_ceph_conf = /etc/ceph/dcn2.conf + store_description = "DCN2 RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + dcn1: + type: edge + replicas: 2 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + *metallb.universe.tf/loadBalancerIPs: <172.17.0.81>* + spec: + type: LoadBalancer + networkAttachments: + - storage + customServiceConfig: | + [DEFAULT] + enabled_import_methods = [web-download,copy-image,glance-direct] + enabled_backends = central:rbd,dcn1:rbd + [glance_store] + default_backend = dcn1 + [central] + rbd_store_ceph_conf = /etc/ceph/central.conf + store_description = "Central RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + [dcn1] + rbd_store_ceph_conf = /etc/ceph/dcn1.conf + store_description = "DCN1 RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + dcn2: + type: edge + replicas: 2 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + *metallb.universe.tf/loadBalancerIPs: <172.17.0.82>* + spec: + type: LoadBalancer + networkAttachments: + - storage + customServiceConfig: | + [DEFAULT] + enabled_import_methods = [web-download,copy-image,glance-direct] + enabled_backends = central:rbd,dcn2:rbd + [glance_store] + default_backend = dcn2 + [central] + rbd_store_ceph_conf = /etc/ceph/central.conf + store_description = "Central RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True + [dcn2] + rbd_store_ceph_conf = /etc/ceph/dcn2.conf + store_description = "DCN2 RBD backend" + rbd_store_pool = images + rbd_store_user = openstack + rbd_thin_provisioning = True +EOF +---- ++ +where: + +`<172.17.0.80>`:: +Specifies the load balancer IP for the central {image_service} API. + +`<172.17.0.81>`:: +Specifies the load balancer IP for the DCN1 edge {image_service} API. + +`<172.17.0.82>`:: +Specifies the load balancer IP for the DCN2 edge {image_service} API. + ++ +You must configure the Compute nodes at each site to use their local {image_service} endpoints. For example, Compute nodes at central use 172.17.0.80, Compute nodes at dcn1 use 172.17.0.81, and Compute nodes at dcn2 use 172.17.0.82. This configuration is applied when you adopt the data plane by adding a per-site ConfigMap with the `glance_api_servers` setting to each `OpenStackDataPlaneNodeSet`. For more information, see xref:adopting-compute-services-to-the-data-plane_{context}[Adopting Compute services to the data plane]. + ++ +[NOTE] +==== +* The central `GlanceAPI` uses `type: split` and has access to all {Ceph} clusters. The `keystoneEndpoint: central` setting registers this API as the public endpoint in the {identity_service}. +* Each edge `GlanceAPI` uses `type: edge` and has access to its local {Ceph} cluster plus the central cluster. This enables image copying between sites. +* Set the `storageRequest` PVC size based on the storage requirements of each edge site. +* Adjust the number of edge sites and their names to match your DCN deployment. +==== + +. Patch the `OpenStackControlPlane` CR to deploy the {image_service} with multiple {Ceph} back ends: ++ +---- +$ oc patch openstackcontrolplane openstack --type=merge --patch-file glance_dcn_patch.yaml +---- + +. Verify that the {image_service} stores are available for each site: ++ +---- +$ glance stores-info ++----------+----------------------------------------------------------------------------------+ +| Property | Value | ++----------+----------------------------------------------------------------------------------+ +| stores | [{"id": "central", "description": "Central RBD backend", "default": "true"}, | +| | {"id": "dcn1", "description": "dcn1 RBD backend"}, {"id": "dcn2", "description": | +| | "dcn2 RBD backend"}] | ++----------+----------------------------------------------------------------------------------+ +---- ++ +The output should list one store for each {Ceph} back end configured in the central `GlanceAPI`, and the central store should be marked as the default. If any stores are missing, check the `customServiceConfig` in the `glanceAPIs` section of the patch and verify that the {Ceph} configuration files are present in the `ceph-conf-central` secret. + +. Verify that image import methods include `copy-image`, which is required for copying images between stores: ++ +---- +$ glance import-info ++----------------+----------------------------------------------------------------------------------+ +| Property | Value | ++----------------+----------------------------------------------------------------------------------+ +| import-methods | {"description": "Import methods available.", "type": "array", "value": ["web- | +| | download", "copy-image", "glance-direct"]} | ++----------------+----------------------------------------------------------------------------------+ +---- + +. Upload a test image to the central store. Note the image ID: ++ +---- +$ glance image-create --disk-format raw --container-format bare --name test-image \ + --file --store central +---- + +. Verify that the image ID from the previous command is shown in the central {CephCluster} cluster's `images` pool: ++ +---- +$ sudo cephadm shell --config /etc/ceph/central.conf --keyring /etc/ceph/central.client.openstack.keyring \ + -- rbd -p images --cluster central ls -l +NAME SIZE PARENT FMT PROT LOCK + 20 MiB 2 +---- + +. Copy the image to an edge site using the `copy-image` import method: ++ +---- +$ glance image-import --stores dcn1 --import-method copy-image +---- + +. After the import completes, verify that the `stores` field on the image now includes both `central` and `dcn1`: ++ +---- +$ glance image-show | grep stores +| stores | central,dcn1 | +---- + +. Verify the image was copied to the DCN1 {CephCluster} cluster: ++ +---- +$ sudo cephadm shell --config /etc/ceph/dcn1.conf --keyring /etc/ceph/dcn1.client.openstack.keyring \ + -- rbd -p images --cluster dcn1 ls -l +NAME SIZE PARENT FMT PROT LOCK + 20 MiB 2 +---- ++ +The image is now present on the DCN1 {CephCluster} cluster, confirming that {image_service} can copy images between sites. Repeat the `glance image-import` command for each additional edge site to distribute the image to all DCN locations. diff --git a/docs_user/modules/proc_adopting-the-block-storage-service.adoc b/docs_user/modules/proc_adopting-the-block-storage-service.adoc index a1ac480fb..a69d0bb8c 100644 --- a/docs_user/modules/proc_adopting-the-block-storage-service.adoc +++ b/docs_user/modules/proc_adopting-the-block-storage-service.adoc @@ -359,3 +359,7 @@ $ openstack --os-volume-api-version 3.47 volume create --backup + [NOTE] You do not boot a {compute_service_first_ref} instance by using the new `volume from` image or try to detach the previous volume because the {compute_service} and the {block_storage} are still not connected. + +.Additional resources + +* xref:adopting-block-storage-service-with-dcn-backend_{context}[Adopting the {block_storage} with multiple {Ceph} back ends (DCN)]. diff --git a/docs_user/modules/proc_configuring-a-ceph-backend.adoc b/docs_user/modules/proc_configuring-a-ceph-backend.adoc index 980c8072a..6fe9e59d3 100644 --- a/docs_user/modules/proc_configuring-a-ceph-backend.adoc +++ b/docs_user/modules/proc_configuring-a-ceph-backend.adoc @@ -74,10 +74,63 @@ stringData: mon_host = 10.1.1.2,10.1.1.3,10.1.1.4 ---- + -* `mon_host` specifies the addresses of the cluster's monitors. If you use IPv6, use brackets for the `mon_host`. For example: -`mon_host = [v2:[fd00:cccc::100]:3300/0,v1:[fd00:cccc::100]:6789/0]` +where: ++ +`mon_host`:: specifies the addresses of the cluster's monitors. If you use IPv6, use brackets for the `mon_host`. For example: `mon_host = [v2:[fd00:cccc::100]:3300/0,v1:[fd00:cccc::100]:6789/0]` ++ +[NOTE] +==== +For Distributed Compute Node (DCN) deployments with multiple {Ceph} clusters, create one secret per site. Each secret contains only the keys that the respective site requires. For more information on the rationale and key distribution pattern, see xref:ceph-migration-dcn_{context}[{Ceph} migration for Distributed Compute Node deployments]. + +The {Ceph} configuration files for all clusters are available on the {OpenStackShort} controller at either `/var/lib/tripleo-config/ceph/`, or `/etc/ceph`. Copy them locally and create the per-site secrets: + +---- +$ CEPH_SSH="ssh root@" +$ CEPH_DIR="/var/lib/tripleo-config/ceph" +$ TMPDIR=$(mktemp -d) + +$ $CEPH_SSH "cat ${CEPH_DIR}/central.conf" > ${TMPDIR}/central.conf +$ $CEPH_SSH "sudo cat ${CEPH_DIR}/central.client.openstack.keyring" > ${TMPDIR}/central.client.openstack.keyring +$ $CEPH_SSH "cat ${CEPH_DIR}/dcn1.conf" > ${TMPDIR}/dcn1.conf +$ $CEPH_SSH "sudo cat ${CEPH_DIR}/dcn1.client.openstack.keyring" > ${TMPDIR}/dcn1.client.openstack.keyring +$ $CEPH_SSH "cat ${CEPH_DIR}/dcn2.conf" > ${TMPDIR}/dcn2.conf +$ $CEPH_SSH "sudo cat ${CEPH_DIR}/dcn2.client.openstack.keyring" > ${TMPDIR}/dcn2.client.openstack.keyring + +# Central site secret: contains all clusters +$ oc create secret generic ceph-conf-central \ + --from-file=${TMPDIR}/central.conf \ + --from-file=${TMPDIR}/central.client.openstack.keyring \ + --from-file=${TMPDIR}/dcn1.conf \ + --from-file=${TMPDIR}/dcn1.client.openstack.keyring \ + --from-file=${TMPDIR}/dcn2.conf \ + --from-file=${TMPDIR}/dcn2.client.openstack.keyring \ + -n openstack + +# DCN1 edge site secret: central + local only +$ oc create secret generic ceph-conf-dcn1 \ + --from-file=${TMPDIR}/central.conf \ + --from-file=${TMPDIR}/central.client.openstack.keyring \ + --from-file=${TMPDIR}/dcn1.conf \ + --from-file=${TMPDIR}/dcn1.client.openstack.keyring \ + -n openstack + +# DCN2 edge site secret: central + local only +$ oc create secret generic ceph-conf-dcn2 \ + --from-file=${TMPDIR}/central.conf \ + --from-file=${TMPDIR}/central.client.openstack.keyring \ + --from-file=${TMPDIR}/dcn2.conf \ + --from-file=${TMPDIR}/dcn2.client.openstack.keyring \ + -n openstack + +$ rm -rf ${TMPDIR} +---- -. In your `OpenStackControlPlane` CR, inject `ceph.conf` and `ceph.client.openstack.keyring` to the {OpenStackShort} services that are defined in the propagation list. For example: +Repeat for each additional edge site. Each edge site secret must include the central cluster files and only the files for that edge site's local cluster. + +When configuring `extraMounts` on the `OpenStackControlPlane`, use propagation labels matching the service instance names (for example, `central`, `dcn1`, `dcn2`) so that each pod mounts only its site-specific secret. +==== + +. In your `OpenStackControlPlane` CR, inject the {Ceph} configuration into the {OpenStackShort} service pods using `extraMounts`. For a single-cluster deployment, propagate one secret to all services: + ---- $ oc patch openstackcontrolplane openstack --type=merge --patch ' @@ -104,3 +157,58 @@ spec: readOnly: true ' ---- ++ +For a DCN deployment with per-site secrets, use propagation labels matching each service instance name so that each pod receives only the keys for its site: ++ +---- +$ oc patch openstackcontrolplane openstack --type=merge --patch ' +spec: + extraMounts: + - name: v1 + region: r1 + extraVol: + - extraVolType: Ceph + propagation: + - central + - CinderBackup + - ManilaShare + volumes: + - name: ceph-central + projected: + sources: + - secret: + name: ceph-conf-central + mounts: + - name: ceph-central + mountPath: "/etc/ceph" + readOnly: true + - extraVolType: Ceph + propagation: + - dcn1 + volumes: + - name: ceph-dcn1 + projected: + sources: + - secret: + name: ceph-conf-dcn1 + mounts: + - name: ceph-dcn1 + mountPath: "/etc/ceph" + readOnly: true + - extraVolType: Ceph + propagation: + - dcn2 + volumes: + - name: ceph-dcn2 + projected: + sources: + - secret: + name: ceph-conf-dcn2 + mounts: + - name: ceph-dcn2 + mountPath: "/etc/ceph" + readOnly: true +' +---- ++ +The propagation label `central` matches the {image_service} and {block_storage} pod instances named `central`. The `CinderBackup` and `ManilaShare` labels are service-type propagation and apply to all {block_storage} backup and {rhos_component_storage_file} pods, which run only at the central site. Replace `central`, `dcn1`, and `dcn2` with the instance names used in your deployment. diff --git a/docs_user/modules/proc_deploying-file-systems-service-control-plane.adoc b/docs_user/modules/proc_deploying-file-systems-service-control-plane.adoc index 9b8dc88c7..7fbe317e1 100644 --- a/docs_user/modules/proc_deploying-file-systems-service-control-plane.adoc +++ b/docs_user/modules/proc_deploying-file-systems-service-control-plane.adoc @@ -6,6 +6,11 @@ [role="_abstract"] Copy the {rhos_component_storage_file_first_ref} configuration from the {rhos_prev_long} ({OpenStackShort}) {rhos_prev_ver} deployment, and then deploy the {rhos_component_storage_file} on the control plane. +[NOTE] +==== +In a {rhos_prev_long} DCN deployment, the {rhos_component_storage_file} runs only at the central site and is not deployed on edge site Compute nodes. After adoption, {rhos_long} supports deploying the {rhos_component_storage_file} at edge sites if required. +==== + .Prerequisites * The {rhos_component_storage_file} systemd services such as `api`, `cron`, and `scheduler` are stopped. For more information, see xref:stopping-openstack-services_migrating-databases[Stopping {rhos_prev_long} services]. @@ -114,6 +119,7 @@ __EOF__ * `share_backend_name` specifies the names of the back ends to use in {rhos_long}. Ensure that the names of the back ends are the same as they were in {OpenStackShort} {rhos_prev_ver}. * `networkAttachments` specifies the appropriate storage management network. For example, the `manilaShares` instance with the CephFS back-end driver is connected to the `storage` network. * `extraMounts` specifies additional files to add to any of the services. For example, when using {Ceph}, you can add the {rhos_component_storage_file} Ceph user's keyring file as well as the `ceph.conf` configuration file. +* In a DCN deployment where {Ceph} configuration files are named by cluster (for example, `central.conf` instead of `ceph.conf`), set `cephfs_conf_path` to the path of the configuration file for the cluster hosting the CephFS back end, and set `cephfs_cluster_name` to the corresponding cluster name. For example, `cephfs_conf_path=/etc/ceph/central.conf` and `cephfs_cluster_name=central`. Using `/etc/ceph/ceph.conf` when that file does not exist in the mounted secret will cause the `manila-share` service to fail with a `conf_read_file` error. + The following example patch file uses CephFS through NFS: + diff --git a/docs_user/modules/proc_stopping-openstack-services.adoc b/docs_user/modules/proc_stopping-openstack-services.adoc index 223bc1152..9ee6785c3 100644 --- a/docs_user/modules/proc_stopping-openstack-services.adoc +++ b/docs_user/modules/proc_stopping-openstack-services.adoc @@ -235,3 +235,154 @@ endif::[] ---- + If the status of each service is `OK`, then the services stopped successfully. + +. For Distributed Compute Node (DCN) deployments where {image_service}, {block_storage}, and {Ceph} services run on edge Compute nodes, stop the {image_service}, {block_storage}, and etcd services on all edge Compute nodes with the `DistributedComputeHCI` role: ++ +[NOTE] +==== +The `DistributedComputeHCI` role runs `GlanceApiEdge`, `CinderVolumeEdge`, and `Etcd` services. A minimum of three nodes per site use this role. Skip this step if your DCN deployment does not run these services on edge Compute nodes. The examples in this procedure use hyper-converged (HCI) roles. If your deployment does not use HCI, the same services apply to the `DistributedCompute` role, which runs the same `GlanceApiEdge`, `CinderVolumeEdge`, and `Etcd` services but without Ceph OSD, Ceph Monitor, or Ceph Manager. +==== ++ +.. Define shell variables for your `DistributedComputeHCI` edge Compute nodes: ++ +[subs=+quotes] +---- +# DCN1 edge site DistributedComputeHCI nodes +DCN1_HCI0_SSH="ssh -i ** root@**" +DCN1_HCI1_SSH="ssh -i ** root@**" +DCN1_HCI2_SSH="ssh -i ** root@**" + +# DCN2 edge site DistributedComputeHCI nodes +DCN2_HCI0_SSH="ssh -i ** root@**" +DCN2_HCI1_SSH="ssh -i ** root@**" +DCN2_HCI2_SSH="ssh -i ** root@**" +---- ++ +where: + +``:: +Specifies the path to your SSH key for each `DistributedComputeHCI` edge Compute node on each DCN edge site. + +``, ``, ``:: +Specifies the IP address for each `DistributedComputeHCI` edge Compute node within the DCN1 edge site. + +``, ``, ``:: +Specifies the IP address for each `DistributedComputeHCI` edge Compute node within the DCN2 edge site. ++ +.. Stop the storage services on all `DistributedComputeHCI` nodes: ++ +---- +# Services to stop on DistributedComputeHCI edge compute nodes +DCN_HCI_SERVICES=("tripleo_glance_api_internal.service" + "tripleo_cinder_volume.service" + "tripleo_etcd.service") + +# List of all DistributedComputeHCI node SSH commands +DCN_HCI_NODES=("$DCN1_HCI0_SSH" + "$DCN1_HCI1_SSH" + "$DCN1_HCI2_SSH" + "$DCN2_HCI0_SSH" + "$DCN2_HCI1_SSH" + "$DCN2_HCI2_SSH") + +echo "Stopping storage services on DistributedComputeHCI nodes" +for node_ssh in "${DCN_HCI_NODES[@]}"; do + [ -z "$node_ssh" ] && continue + echo "Processing node: $node_ssh" + for service in "${DCN_HCI_SERVICES[@]}"; do + if $node_ssh sudo systemctl is-active $service 2>/dev/null; then + echo "Stopping $service" + $node_ssh sudo systemctl stop $service + fi + done +done + +echo "Checking storage services on DistributedComputeHCI nodes" +for node_ssh in "${DCN_HCI_NODES[@]}"; do + [ -z "$node_ssh" ] && continue + for service in "${DCN_HCI_SERVICES[@]}"; do + if ! $node_ssh systemctl show $service 2>/dev/null | grep ActiveState=inactive >/dev/null; then + echo "ERROR: Service $service still running on $node_ssh" + else + echo "OK: Service $service is not running on $node_ssh" + fi + done +done +---- ++ +[NOTE] +==== +* On edge sites, the {image_service} runs with the service name `tripleo_glance_api_internal.service`, which is different from the `tripleo_glance_api.service` on the central controller. +* The {block_storage} volume service (`tripleo_cinder_volume.service`) uses the same service name on both edge sites and the central controller. +* The etcd service (`tripleo_etcd.service`) is used as a distributed lock manager (DLM) for the {block_storage} volume service running in active/active mode on edge sites. +==== + +. If your DCN deployment includes `DistributedComputeHCIScaleOut` nodes, stop the HAProxy service on those nodes: ++ +[NOTE] +==== +The `DistributedComputeHCIScaleOut` role is used to scale compute and storage capacity beyond the initial three `DistributedComputeHCI` nodes at each site. These nodes run `HAProxyEdge`, which proxies {image_service} requests to the `GlanceApiEdge` instances on `DistributedComputeHCI` nodes. Skip this step if your DCN deployment does not include `DistributedComputeHCIScaleOut` nodes. For non-HCI deployments, the equivalent role is `DistributedComputeScaleOut`, which runs the same `HAProxyEdge` service. +==== ++ +.. Define shell variables for your `DistributedComputeHCIScaleOut` edge Compute nodes: ++ +[subs=+quotes] +---- +# DCN1 edge site DistributedComputeHCIScaleOut nodes +DCN1_SCALEOUT0_SSH="ssh -i ** root@**" +DCN1_SCALEOUT1_SSH="ssh -i ** root@**" + +# DCN2 edge site DistributedComputeHCIScaleOut nodes +DCN2_SCALEOUT0_SSH="ssh -i ** root@**" +DCN2_SCALEOUT1_SSH="ssh -i ** root@**" +---- ++ +where: + +``, ``:: +Specifies the IP address for each `DistributedComputeHCIScaleOut` node within the DCN1 edge site. + +``, ``:: +Specifies the IP address for each `DistributedComputeHCIScaleOut` node within the DCN2 edge site. ++ +.. Stop the services on all `DistributedComputeHCIScaleOut` nodes: ++ +---- +# Services to stop on DistributedComputeHCIScaleOut edge compute nodes +DCN_SCALEOUT_SERVICES=("tripleo_haproxy_edge.service") + +# List of all DistributedComputeHCIScaleOut node SSH commands +DCN_SCALEOUT_NODES=("$DCN1_SCALEOUT0_SSH" + "$DCN1_SCALEOUT1_SSH" + "$DCN2_SCALEOUT0_SSH" + "$DCN2_SCALEOUT1_SSH") + +echo "Stopping services on DistributedComputeHCIScaleOut nodes" +for node_ssh in "${DCN_SCALEOUT_NODES[@]}"; do + [ -z "$node_ssh" ] && continue + echo "Processing node: $node_ssh" + for service in "${DCN_SCALEOUT_SERVICES[@]}"; do + if $node_ssh sudo systemctl is-active $service 2>/dev/null; then + echo "Stopping $service" + $node_ssh sudo systemctl stop $service + fi + done +done + +echo "Checking services on DistributedComputeHCIScaleOut nodes" +for node_ssh in "${DCN_SCALEOUT_NODES[@]}"; do + [ -z "$node_ssh" ] && continue + for service in "${DCN_SCALEOUT_SERVICES[@]}"; do + if ! $node_ssh systemctl show $service 2>/dev/null | grep ActiveState=inactive >/dev/null; then + echo "ERROR: Service $service still running on $node_ssh" + else + echo "OK: Service $service is not running on $node_ssh" + fi + done +done +---- ++ +[NOTE] +==== +* The HAProxy edge service (`tripleo_haproxy_edge.service`) provided a local {image_service} endpoint on `DistributedComputeHCIScaleOut` nodes, proxying requests to the `GlanceApiEdge` instances on `DistributedComputeHCI` nodes. During adoption, {rhocp_long} Kubernetes service endpoints backed by MetalLB replace HAProxy. +==== diff --git a/scenarios/dcn_storage.yaml b/scenarios/dcn_storage.yaml new file mode 100644 index 000000000..71e11bf07 --- /dev/null +++ b/scenarios/dcn_storage.yaml @@ -0,0 +1,306 @@ +--- +# DCN Storage Adoption Scenario +# Deploys 3 Ceph clusters: Central (az0), DCN1 (az1), DCN2 (az2) +# Each cluster runs on 3 HCI compute nodes + +undercloud: + config: + - section: DEFAULT + option: undercloud_hostname + value: undercloud.example.com + - section: DEFAULT + option: undercloud_timezone + value: UTC + - section: DEFAULT + option: undercloud_debug + value: true + - section: DEFAULT + option: container_cli + value: podman + - section: DEFAULT + option: undercloud_enable_selinux + value: false + - section: DEFAULT + option: generate_service_certificate + value: false + + - section: DEFAULT + option: local_interface + value: eth0 + - section: DEFAULT + option: local_ip + value: 192.168.122.100/24 + - section: DEFAULT + option: enable_routed_networks + value: true + - section: DEFAULT + option: subnets + value: ctlplane-subnet,leaf1,leaf2 + - section: DEFAULT + option: local_subnet + value: ctlplane-subnet + + - section: ctlplane-subnet + option: masquerade + value: false + + - section: leaf1 + option: cidr + value: 192.168.133.0/24 + - section: leaf1 + option: gateway + value: 192.168.133.1 + - section: leaf1 + option: masquerade + value: false + - section: leaf1 + option: inspection_iprange + value: 192.168.133.200,192.168.133.220 + + - section: leaf2 + option: cidr + value: 192.168.144.0/24 + - section: leaf2 + option: gateway + value: 192.168.144.1 + - section: leaf2 + option: masquerade + value: false + - section: leaf2 + option: inspection_iprange + value: 192.168.144.200,192.168.144.220 + undercloud_parameters_override: "dcn_storage/hieradata_overrides_undercloud.yaml" + undercloud_parameters_defaults: "dcn_storage/undercloud_parameter_defaults.yaml" + ctlplane_vip: 192.168.122.98 +cloud_domain: "example.com" +hostname_groups_map: + # map ansible groups in the inventory to role hostname format for + # 17.1 deployment + osp-computes: "central-computehci" + osp-controllers: "central-controller" + osp-dcn1-compute-az1s: "dcn1-distributedcomputehci" + osp-dcn2-compute-az2s: "dcn2-distributedcomputehci" + + +roles_groups_map: + # map ansible groups to tripleo Role names + osp-computes: "ComputeHCI" + osp-controllers: "Controller" + osp-dcn1-compute-az1s: "DistributedComputeHCI" + osp-dcn2-compute-az2s: "DistributedComputeHCI" + +network_tripleo_network_map: + ctlplane: ctlplane + ctlplanedcn1: ctlplane + ctlplanedcn2: ctlplane + internalapi: internalapi + internalapidcn1: internalapi + internalapidcn2: internalapi + storage: storage + storagedcn1: storage + storagedcn2: storage + storagemgmt: storagemgmt + storagemgmtdcn1: storagemgmt + storagemgmtdcn2: storagemgmt + tenant: tenant + tenantdcn1: tenant + tenantdcn2: tenant + +stacks: + - stackname: "central" + args: + - "--override-ansible-cfg /home/zuul/ansible_config.cfg" + - "--templates /usr/share/openstack-tripleo-heat-templates" + - "--libvirt-type qemu" + - "--timeout 90" + - "--overcloud-ssh-user zuul" + - "--deployed-server" + - "--validation-warnings-fatal" + - "--disable-validations" + - "--heat-type pod" + - "--disable-protected-resource-types" + vars: + - "/home/zuul/deployed_ceph_central.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/docker-ha.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/podman.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/low-memory-usage.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/debug.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/manila-cephfsnative-config.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/cephadm/ceph-mds.yaml" + network_data_file: "dcn_storage/central/network_data.yaml.j2" + vips_data_file: "dcn_storage/central/vips_data.yaml" + roles_file: "dcn_storage/central/roles.yaml" + ceph_osd_spec_file: "dcn_storage/ceph/osd_spec.yaml" + config_download_file: "dcn_storage/central/config_download.yaml" + stack_nodes: + - osp-computes + - osp-controllers + ctlplanenet: ctlplane + routes: + - ip_netmask: 0.0.0.0/0 + next_hop: 192.168.122.1 + default: true + network_routes: + internalapi: + - ip_netmask: 172.17.10.0/24 + next_hop: 172.17.0.1 + - ip_netmask: 172.17.20.0/24 + next_hop: 172.17.0.1 + storage: + - ip_netmask: 172.18.10.0/24 + next_hop: 172.18.0.1 + - ip_netmask: 172.18.20.0/24 + next_hop: 172.18.0.1 + tenant: + - ip_netmask: 172.19.10.0/24 + next_hop: 172.19.0.1 + - ip_netmask: 172.19.20.0/24 + next_hop: 172.19.0.1 + pre_oc_run: + - name: Deploy Central Ceph + type: playbook + source: "adoption_deploy_ceph.yml" + extra_vars: + stack_index: 0 + post_oc_run: + - name: Export central stack for DCN + type: playbook + source: "adoption_dcn_export.yml" + extra_vars: + stack_name: central + - stackname: dcn1 + args: + - "--override-ansible-cfg /home/zuul/ansible_config.cfg" + - "--templates /usr/share/openstack-tripleo-heat-templates" + - "--libvirt-type qemu" + - "--timeout 90" + - "--overcloud-ssh-user zuul" + - "--deployed-server" + - "--validation-warnings-fatal" + - "--disable-validations" + - "--heat-type pod" + - "--disable-protected-resource-types" + vars: + - "/home/zuul/deployed_ceph_dcn1.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/docker-ha.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/podman.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/low-memory-usage.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/debug.yaml" + - "/home/zuul/overcloud-deploy/central/central-export.yaml" + - "/home/zuul/central_ceph_external.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm-rbd-only.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/dcn-storage.yaml" + - "/home/zuul/glance_dcn1.yaml" + additional_files: + - "dcn_storage/dcn1/glance_dcn1.yaml" + network_data_file: "dcn_storage/dcn1/network_data.yaml.j2" + vips_data_file: "dcn_storage/dcn1/vips_data.yaml" + roles_file: "dcn_storage/dcn1/roles.yaml" + ceph_osd_spec_file: "dcn_storage/ceph/osd_spec.yaml" + config_download_file: "dcn_storage/dcn1/config_download.yaml" + stack_nodes: + - osp-dcn1-compute-az1s + ctlplanenet: ctlplanedcn1 + routes: + - ip_netmask: 0.0.0.0/0 + next_hop: 192.168.133.1 + default: true + network_routes: + internalapidcn1: + - ip_netmask: 172.17.0.0/24 + next_hop: 172.17.10.1 + - ip_netmask: 172.17.20.0/24 + next_hop: 172.17.10.1 + storagedcn1: + - ip_netmask: 172.18.0.0/24 + next_hop: 172.18.10.1 + - ip_netmask: 172.18.20.0/24 + next_hop: 172.18.10.1 + storagemgmtdcn1: + - ip_netmask: 172.20.0.0/24 + next_hop: 172.20.10.1 + - ip_netmask: 172.20.20.0/24 + next_hop: 172.20.10.1 + tenantdcn1: + - ip_netmask: 172.19.0.0/24 + next_hop: 172.19.10.1 + - ip_netmask: 172.19.20.0/24 + next_hop: 172.19.10.1 + pre_oc_run: + - name: Deploy DCN1 Ceph + type: playbook + source: "adoption_deploy_ceph.yml" + extra_vars: + stack_index: 1 + - stackname: dcn2 + args: + - "--override-ansible-cfg /home/zuul/ansible_config.cfg" + - "--templates /usr/share/openstack-tripleo-heat-templates" + - "--libvirt-type qemu" + - "--timeout 90" + - "--overcloud-ssh-user zuul" + - "--deployed-server" + - "--validation-warnings-fatal" + - "--disable-validations" + - "--heat-type pod" + - "--disable-protected-resource-types" + vars: + - "/home/zuul/deployed_ceph_dcn2.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/docker-ha.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/podman.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/low-memory-usage.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/debug.yaml" + - "/home/zuul/overcloud-deploy/central/central-export.yaml" + - "/home/zuul/central_ceph_external.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm-rbd-only.yaml" + - "/usr/share/openstack-tripleo-heat-templates/environments/dcn-storage.yaml" + - "/home/zuul/glance_dcn2.yaml" + additional_files: + - "dcn_storage/dcn2/glance_dcn2.yaml" + network_data_file: "dcn_storage/dcn2/network_data.yaml.j2" + vips_data_file: "dcn_storage/dcn2/vips_data.yaml" + roles_file: "dcn_storage/dcn2/roles.yaml" + ceph_osd_spec_file: "dcn_storage/ceph/osd_spec.yaml" + config_download_file: "dcn_storage/dcn2/config_download.yaml" + stack_nodes: + - osp-dcn2-compute-az2s + ctlplanenet: ctlplanedcn2 + routes: + - ip_netmask: 0.0.0.0/0 + next_hop: 192.168.144.1 + default: true + network_routes: + internalapidcn2: + - ip_netmask: 172.17.0.0/24 + next_hop: 172.17.20.1 + - ip_netmask: 172.17.10.0/24 + next_hop: 172.17.20.1 + storagedcn2: + - ip_netmask: 172.18.0.0/24 + next_hop: 172.18.20.1 + - ip_netmask: 172.18.10.0/24 + next_hop: 172.18.20.1 + storagemgmtdcn2: + - ip_netmask: 172.20.0.0/24 + next_hop: 172.20.20.1 + - ip_netmask: 172.20.10.0/24 + next_hop: 172.20.20.1 + tenantdcn2: + - ip_netmask: 172.19.0.0/24 + next_hop: 172.19.20.1 + - ip_netmask: 172.19.10.0/24 + next_hop: 172.19.20.1 + pre_oc_run: + - name: Deploy DCN2 Ceph + type: playbook + source: "adoption_deploy_ceph.yml" + extra_vars: + stack_index: 2 + post_oc_run: + - name: Update central with DCN Ceph stores + type: playbook + source: "adoption_dcn_update_central.yml" + extra_vars: + central_stack_name: central + dcn_stack_names: "dcn1,dcn2" diff --git a/scenarios/dcn_storage/central/config_download.yaml b/scenarios/dcn_storage/central/config_download.yaml new file mode 100644 index 000000000..6c7d28a5b --- /dev/null +++ b/scenarios/dcn_storage/central/config_download.yaml @@ -0,0 +1,117 @@ +--- +resource_registry: + # yamllint disable rule:line-length + OS::TripleO::DeployedServer::ControlPlanePort: /usr/share/openstack-tripleo-heat-templates/deployed-server/deployed-neutron-port.yaml + OS::TripleO::OVNMacAddressNetwork: OS::Heat::None + OS::TripleO::OVNMacAddressPort: OS::Heat::None + OS::TripleO::Compute::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml + OS::TripleO::Compute::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml + OS::TripleO::Compute::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml + OS::TripleO::Compute::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml + OS::TripleO::Controller::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml + OS::TripleO::Controller::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml + OS::TripleO::Controller::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml + OS::TripleO::Controller::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml + OS::TripleO::Controller::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml + OS::TripleO::Services::CeilometerAgentCentral: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-central-container-puppet.yaml + OS::TripleO::Services::CeilometerAgentNotification: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-notification-container-puppet.yaml + OS::TripleO::Services::CeilometerAgentIpmi: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-ipmi-container-puppet.yaml + OS::TripleO::Services::ComputeCeilometerAgent: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-compute-container-puppet.yaml + OS::TripleO::Services::Collectd: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/collectd-container-puppet.yaml + OS::TripleO::Services::MetricsQdr: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/qdr-container-puppet.yaml + OS::TripleO::Services::OsloMessagingRpc: /usr/share/openstack-tripleo-heat-templates/deployment/rabbitmq/rabbitmq-messaging-rpc-pacemaker-puppet.yaml + OS::TripleO::Services::OsloMessagingNotify: /usr/share/openstack-tripleo-heat-templates/deployment/rabbitmq/rabbitmq-messaging-notify-shared-puppet.yaml + OS::TripleO::Services::HAproxy: /usr/share/openstack-tripleo-heat-templates/deployment/haproxy/haproxy-pacemaker-puppet.yaml + OS::TripleO::Services::Pacemaker: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/pacemaker-baremetal-puppet.yaml + OS::TripleO::Services::PacemakerRemote: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/pacemaker-remote-baremetal-puppet.yaml + OS::TripleO::Services::Clustercheck: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/clustercheck-container-puppet.yaml + OS::TripleO::Services::Redis: /usr/share/openstack-tripleo-heat-templates/deployment/database/redis-pacemaker-puppet.yaml + OS::TripleO::Services::Rsyslog: /usr/share/openstack-tripleo-heat-templates/deployment/logging/rsyslog-container-puppet.yaml + OS::TripleO::Services::MySQL: /usr/share/openstack-tripleo-heat-templates/deployment/database/mysql-pacemaker-puppet.yaml + OS::TripleO::Services::CinderBackup: /usr/share/openstack-tripleo-heat-templates/deployment/cinder/cinder-backup-pacemaker-puppet.yaml + OS::TripleO::Services::CinderVolume: /usr/share/openstack-tripleo-heat-templates/deployment/cinder/cinder-volume-pacemaker-puppet.yaml + OS::TripleO::Services::HeatApi: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-container-puppet.yaml + OS::TripleO::Services::HeatApiCfn: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-cfn-container-puppet.yaml + OS::TripleO::Services::HeatApiCloudwatch: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-cloudwatch-disabled-puppet.yaml + OS::TripleO::Services::HeatEngine: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-engine-container-puppet.yaml + OS::TripleO::Services::NovaAZConfig: /usr/share/openstack-tripleo-heat-templates/deployment/nova/nova-az-config.yaml +parameter_defaults: + # DCN deployment requires loose rp_filter to allow asymmetric routing + KernelIpv4ConfAllRpFilter: 2 + RedisVirtualFixedIPs: + - ip_address: 192.168.122.110 + use_neutron: false + OVNDBsVirtualFixedIPs: + - ip_address: 192.168.122.111 + use_neutron: false + ControllerExtraConfig: + nova::availability_zone::default_schedule_zone: az-central + nova::compute::libvirt::services::libvirt_virt_type: qemu + nova::compute::libvirt::virt_type: qemu + ComputeExtraConfig: + nova::compute::libvirt::services::libvirt_virt_type: qemu + nova::compute::libvirt::virt_type: qemu + BarbicanSimpleCryptoGlobalDefault: true + Debug: true + DockerPuppetDebug: true + ContainerCli: podman + ControllerCount: 1 + ComputeHCICount: 3 + NeutronGlobalPhysnetMtu: 1350 + CinderLVMLoopDeviceSize: 20480 + CloudName: overcloud.example.com + CloudNameInternal: overcloud.internalapi.example.com + CloudNameStorage: overcloud.storage.example.com + CloudNameStorageManagement: overcloud.storagemgmt.example.com + CloudNameCtlplane: overcloud.ctlplane.example.com + CloudDomain: example.com + NetworkConfigWithAnsible: false + ControllerHostnameFormat: '%stackname%-controller-%index%' + ComputeHCIHostnameFormat: '%stackname%-computehci-%index%' + CtlplaneNetworkAttributes: + network: + dns_domain: example.com + mtu: 1500 + name: ctlplane + tags: + - 192.168.122.0/24 + subnets: + ctlplane-subnet: + cidr: 192.168.122.0/24 + dns_nameservers: 192.168.122.10 + gateway_ip: 192.168.122.10 + host_routes: + - destination: 192.168.133.0/24 # Route to DCN1 + nexthop: 192.168.122.10 + - destination: 192.168.144.0/24 # Route to DCN2 + nexthop: 192.168.122.10 + name: ctlplane-subnet + ip_version: 4 + + BondInterfaceOvsOptions: bond_mode=active-backup + ComputeHCINetworkConfigTemplate: /home/stack/central/network/spine-leaf-nics/compute0.j2 + ComputeHCIParameters: + NeutronBridgeMappings: + - leaf0:br-ex + ControllerNetworkConfigTemplate: /home/stack/central/network/spine-leaf-nics/controller0.j2 + ControllerParameters: + NeutronBridgeMappings: + - leaf0:br-ex + DnsServers: + - 10.11.5.19 + - 10.5.30.160 + NeutronEnableDVR: 'false' + NeutronFlatNetworks: leaf0,leaf1,leaf2 + NeutronNetworkType: geneve + NeutronNetworkVLANRanges: leaf0:1:1000,leaf1:1:1000,leaf2:1:1000 + NeutronTunnelTypes: geneve + NovaComputeAvailabilityZone: az-central + CinderStorageAvailabilityZone: az-central + NovaCrossAZAttach: false + NtpServer: + - clock.redhat.com + - clock2.redhat.com + PythonInterpreter: /usr/bin/python3 + RootStackName: central + VipSubnetMap: + ctlplane: ctlplane-subnet diff --git a/scenarios/dcn_storage/central/network_data.yaml.j2 b/scenarios/dcn_storage/central/network_data.yaml.j2 new file mode 100644 index 000000000..8541dd86d --- /dev/null +++ b/scenarios/dcn_storage/central/network_data.yaml.j2 @@ -0,0 +1,107 @@ +--- +- name: Storage + mtu: 1496 + vip: true + name_lower: storage + dns_domain: storage.{{ cloud_domain }}. + service_net_map_replace: storage + subnets: + storage_subnet: + vlan: 21 + ip_subnet: '172.18.0.0/24' + allocation_pools: [{'start': '172.18.0.120', 'end': '172.18.0.250'}] + routes: + - destination: 172.18.10.0/24 # Route to dcn1 + nexthop: 172.18.0.1 + - destination: 172.18.20.0/24 # Route to dcn2 + nexthop: 172.18.0.1 + storage_leaf1: + vlan: 31 + ip_subnet: '172.18.10.0/24' + allocation_pools: [{'start': '172.18.10.120', 'end': '172.18.10.250'}] + storage_leaf2: + vlan: 41 + ip_subnet: '172.18.20.0/24' + allocation_pools: [{'start': '172.18.20.120', 'end': '172.18.20.250'}] + +- name: StorageMgmt + mtu: 1496 + vip: true + name_lower: storage_mgmt + dns_domain: storagemgmt.{{ cloud_domain }}. + service_net_map_replace: storage_mgmt + subnets: + storage_mgmt_subnet: + vlan: 23 + ip_subnet: '172.20.0.0/24' + allocation_pools: [{'start': '172.20.0.120', 'end': '172.20.0.250'}] + storage_mgmt_leaf1: + vlan: 33 + ip_subnet: '172.20.10.0/24' + allocation_pools: [{'start': '172.20.10.120', 'end': '172.20.10.250'}] + storage_mgmt_leaf2: + vlan: 43 + ip_subnet: '172.20.20.0/24' + allocation_pools: [{'start': '172.20.20.120', 'end': '172.20.20.250'}] + +- name: InternalApi + mtu: 1496 + vip: true + name_lower: internal_api + dns_domain: internal-api.{{ cloud_domain }}. + service_net_map_replace: internal_api + subnets: + internal_api_subnet: + vlan: 20 + ip_subnet: '172.17.0.0/24' + allocation_pools: [{'start': '172.17.0.120', 'end': '172.17.0.250'}] + routes: + - destination: 172.17.10.0/24 # Route to dcn1 + nexthop: 172.17.0.1 + - destination: 172.17.20.0/24 # Route to dcn2 + nexthop: 172.17.0.1 + internal_api_leaf1: + vlan: 30 + ip_subnet: '172.17.10.0/24' + allocation_pools: [{'start': '172.17.10.120', 'end': '172.17.10.250'}] + internal_api_leaf2: + vlan: 40 + ip_subnet: '172.17.20.0/24' + allocation_pools: [{'start': '172.17.20.120', 'end': '172.17.20.250'}] + +- name: Tenant + mtu: 1496 + vip: false # Tenant network does not use VIPs + name_lower: tenant + dns_domain: tenant.{{ cloud_domain }}. + service_net_map_replace: tenant + subnets: + tenant_subnet: + vlan: 22 + ip_subnet: '172.19.0.0/24' + allocation_pools: [{'start': '172.19.0.120', 'end': '172.19.0.250'}] + routes: + - destination: 172.19.10.0/24 # Route to dcn1 + nexthop: 172.19.0.1 + - destination: 172.19.20.0/24 # Route to dcn2 + nexthop: 172.19.0.1 + tenant_leaf_1: + vlan: 32 + ip_subnet: '172.19.10.0/24' + allocation_pools: [{'start': '172.19.10.120', 'end': '172.19.10.250'}] + tenant_leaf_2: + vlan: 42 + ip_subnet: '172.19.20.0/24' + allocation_pools: [{'start': '172.19.20.120', 'end': '172.19.20.250'}] + +- name: External + mtu: 1500 + vip: true + name_lower: external + dns_domain: external.{{ cloud_domain }}. + service_net_map_replace: external + subnets: + external_subnet: + vlan: 44 + ip_subnet: '10.0.0.0/24' + allocation_pools: [{'start': '10.0.0.150', 'end': '10.0.0.250'}] diff --git a/scenarios/dcn_storage/central/roles.yaml b/scenarios/dcn_storage/central/roles.yaml new file mode 100644 index 000000000..d3eaeeff0 --- /dev/null +++ b/scenarios/dcn_storage/central/roles.yaml @@ -0,0 +1,256 @@ +############################################################################### +# Role: Controller # +############################################################################### +- name: Controller + description: | + Controller role that has all the controler services loaded and handles + Database, Messaging and Network functions. + CountDefault: 1 + tags: + - primary + - controller + # Create external Neutron bridge for SNAT (and floating IPs when using + # ML2/OVS without DVR) + - external_bridge + networks: + External: + subnet: external_subnet + InternalApi: + subnet: internal_api_subnet + Storage: + subnet: storage_subnet + StorageMgmt: + subnet: storage_mgmt_subnet + Tenant: + subnet: tenant_subnet + # For systems with both IPv4 and IPv6, you may specify a gateway network for + # each, such as ['ControlPlane', 'External'] + default_route_networks: ['ControlPlane'] + HostnameFormatDefault: '%stackname%-controller-%index%' + RoleParametersDefault: + OVNCMSOptions: "enable-chassis-as-gw" + # Deprecated & backward-compatible values (FIXME: Make parameters consistent) + # Set uses_deprecated_params to True if any deprecated params are used. + uses_deprecated_params: true + deprecated_param_extraconfig: 'controllerExtraConfig' + deprecated_param_flavor: 'OvercloudControlFlavor' + deprecated_param_image: 'controllerImage' + deprecated_nic_config_name: 'controller.yaml' + update_serial: 1 + ServicesDefault: + - OS::TripleO::Services::Aide + - OS::TripleO::Services::AodhApi + - OS::TripleO::Services::AodhEvaluator + - OS::TripleO::Services::AodhListener + - OS::TripleO::Services::AodhNotifier + - OS::TripleO::Services::AuditD + - OS::TripleO::Services::BarbicanApi + - OS::TripleO::Services::BarbicanBackendSimpleCrypto + - OS::TripleO::Services::BarbicanBackendDogtag + - OS::TripleO::Services::BarbicanBackendKmip + - OS::TripleO::Services::BarbicanBackendPkcs11Crypto + - OS::TripleO::Services::BootParams + - OS::TripleO::Services::CACerts + - OS::TripleO::Services::CeilometerAgentCentral + - OS::TripleO::Services::CeilometerAgentNotification + - OS::TripleO::Services::CephClient + - OS::TripleO::Services::CephExternal + - OS::TripleO::Services::CephGrafana + - OS::TripleO::Services::CephMds + - OS::TripleO::Services::CephMgr + - OS::TripleO::Services::CephMon + - OS::TripleO::Services::CephRbdMirror + - OS::TripleO::Services::CephRgw + - OS::TripleO::Services::CinderApi + - OS::TripleO::Services::CinderBackendDellSc + - OS::TripleO::Services::CinderBackendDellEMCPowerFlex + - OS::TripleO::Services::CinderBackendDellEMCPowermax + - OS::TripleO::Services::CinderBackendDellEMCPowerStore + - OS::TripleO::Services::CinderBackendDellEMCSc + - OS::TripleO::Services::CinderBackendDellEMCUnity + - OS::TripleO::Services::CinderBackendDellEMCVMAXISCSI + - OS::TripleO::Services::CinderBackendDellEMCVNX + - OS::TripleO::Services::CinderBackendDellEMCVxFlexOS + - OS::TripleO::Services::CinderBackendDellEMCXtremio + - OS::TripleO::Services::CinderBackendNetApp + - OS::TripleO::Services::CinderBackendPure + - OS::TripleO::Services::CinderBackendScaleIO + - OS::TripleO::Services::CinderBackendNVMeOF + - OS::TripleO::Services::CinderBackup + - OS::TripleO::Services::CinderHPELeftHandISCSI + - OS::TripleO::Services::CinderScheduler + - OS::TripleO::Services::CinderVolume + - OS::TripleO::Services::Clustercheck + - OS::TripleO::Services::Collectd + - OS::TripleO::Services::ContainerImagePrepare + - OS::TripleO::Services::DesignateApi + - OS::TripleO::Services::DesignateCentral + - OS::TripleO::Services::DesignateProducer + - OS::TripleO::Services::DesignateWorker + - OS::TripleO::Services::DesignateMDNS + - OS::TripleO::Services::DesignateSink + - OS::TripleO::Services::DesignateBind + - OS::TripleO::Services::Etcd + - OS::TripleO::Services::ExternalSwiftProxy + - OS::TripleO::Services::Frr + - OS::TripleO::Services::GlanceApi + - OS::TripleO::Services::GlanceApiInternal + - OS::TripleO::Services::GnocchiApi + - OS::TripleO::Services::GnocchiMetricd + - OS::TripleO::Services::GnocchiStatsd + - OS::TripleO::Services::HAproxy + - OS::TripleO::Services::HeatApi + - OS::TripleO::Services::HeatApiCloudwatch + - OS::TripleO::Services::HeatApiCfn + - OS::TripleO::Services::HeatEngine + - OS::TripleO::Services::Horizon + - OS::TripleO::Services::IpaClient + - OS::TripleO::Services::Ipsec + - OS::TripleO::Services::IronicApi + - OS::TripleO::Services::IronicConductor + - OS::TripleO::Services::IronicInspector + - OS::TripleO::Services::IronicPxe + - OS::TripleO::Services::IronicNeutronAgent + - OS::TripleO::Services::Iscsid + - OS::TripleO::Services::Kernel + - OS::TripleO::Services::Keystone + - OS::TripleO::Services::LoginDefs + - OS::TripleO::Services::ManilaApi + - OS::TripleO::Services::ManilaBackendCephFs + - OS::TripleO::Services::ManilaBackendIsilon + - OS::TripleO::Services::ManilaBackendNetapp + - OS::TripleO::Services::ManilaBackendUnity + - OS::TripleO::Services::ManilaBackendVNX + - OS::TripleO::Services::ManilaBackendVMAX + - OS::TripleO::Services::ManilaScheduler + - OS::TripleO::Services::ManilaShare + - OS::TripleO::Services::Memcached + - OS::TripleO::Services::MetricsQdr + - OS::TripleO::Services::Multipathd + - OS::TripleO::Services::MySQL + - OS::TripleO::Services::MySQLClient + - OS::TripleO::Services::NeutronApi + - OS::TripleO::Services::NeutronBgpVpnApi + - OS::TripleO::Services::NeutronSfcApi + - OS::TripleO::Services::NeutronCorePlugin + - OS::TripleO::Services::NeutronDhcpAgent + - OS::TripleO::Services::NeutronL2gwAgent + - OS::TripleO::Services::NeutronL2gwApi + - OS::TripleO::Services::NeutronL3Agent + - OS::TripleO::Services::NeutronLinuxbridgeAgent + - OS::TripleO::Services::NeutronMetadataAgent + - OS::TripleO::Services::NeutronOvsAgent + - OS::TripleO::Services::NeutronVppAgent + - OS::TripleO::Services::NeutronAgentsIBConfig + - OS::TripleO::Services::NovaApi + - OS::TripleO::Services::NovaConductor + - OS::TripleO::Services::NovaIronic + - OS::TripleO::Services::NovaMetadata + - OS::TripleO::Services::NovaScheduler + - OS::TripleO::Services::NovaVncProxy + - OS::TripleO::Services::ContainersLogrotateCrond + - OS::TripleO::Services::OctaviaApi + - OS::TripleO::Services::OctaviaDeploymentConfig + - OS::TripleO::Services::OctaviaHealthManager + - OS::TripleO::Services::OctaviaHousekeeping + - OS::TripleO::Services::OctaviaWorker + - OS::TripleO::Services::OpenStackClients + - OS::TripleO::Services::OVNDBs + - OS::TripleO::Services::OVNController + - OS::TripleO::Services::Pacemaker + - OS::TripleO::Services::PlacementApi + - OS::TripleO::Services::OsloMessagingRpc + - OS::TripleO::Services::OsloMessagingNotify + - OS::TripleO::Services::Podman + - OS::TripleO::Services::Redis + - OS::TripleO::Services::Rhsm + - OS::TripleO::Services::Rsyslog + - OS::TripleO::Services::RsyslogSidecar + - OS::TripleO::Services::Securetty + - OS::TripleO::Services::Snmp + - OS::TripleO::Services::Sshd + - OS::TripleO::Services::SwiftProxy + - OS::TripleO::Services::SwiftDispersion + - OS::TripleO::Services::SwiftRingBuilder + - OS::TripleO::Services::SwiftStorage + - OS::TripleO::Services::Timesync + - OS::TripleO::Services::Timezone + - OS::TripleO::Services::TripleoFirewall + - OS::TripleO::Services::TripleoPackages + - OS::TripleO::Services::Tuned + - OS::TripleO::Services::Unbound + - OS::TripleO::Services::Vpp +############################################################################### +# Role: ComputeHCI # +############################################################################### +- name: ComputeHCI + description: | + Compute Node role hosting Ceph OSD too + CountDefault: 1 + tags: + - compute + networks: + InternalApi: + subnet: internal_api_subnet + Tenant: + subnet: tenant_subnet + Storage: + subnet: storage_subnet + StorageMgmt: + subnet: storage_mgmt_subnet + default_route_networks: ['ControlPlane'] + HostnameFormatDefault: '%stackname%-computehci-%index%' + RoleParametersDefault: + FsAioMaxNumber: 1048576 + TunedProfileName: "throughput-performance" + NovaComputeStartupDelay: 180 + # CephOSD present so serial has to be 1 + update_serial: 1 + ServicesDefault: + - OS::TripleO::Services::Aide + - OS::TripleO::Services::AuditD + - OS::TripleO::Services::BootParams + - OS::TripleO::Services::CACerts + - OS::TripleO::Services::CephClient + - OS::TripleO::Services::CephExternal + - OS::TripleO::Services::CephOSD + - OS::TripleO::Services::Collectd + - OS::TripleO::Services::ComputeCeilometerAgent + - OS::TripleO::Services::CeilometerAgentIpmi + - OS::TripleO::Services::ComputeNeutronCorePlugin + - OS::TripleO::Services::ComputeNeutronL3Agent + - OS::TripleO::Services::ComputeNeutronMetadataAgent + - OS::TripleO::Services::ComputeNeutronOvsAgent + - OS::TripleO::Services::Frr + - OS::TripleO::Services::IpaClient + - OS::TripleO::Services::Ipsec + - OS::TripleO::Services::Iscsid + - OS::TripleO::Services::Kernel + - OS::TripleO::Services::LoginDefs + - OS::TripleO::Services::MetricsQdr + - OS::TripleO::Services::Multipathd + - OS::TripleO::Services::MySQLClient + - OS::TripleO::Services::NeutronBgpVpnBagpipe + - OS::TripleO::Services::NeutronLinuxbridgeAgent + - OS::TripleO::Services::NeutronVppAgent + - OS::TripleO::Services::NovaAZConfig + - OS::TripleO::Services::NovaCompute + - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaLibvirtGuests + - OS::TripleO::Services::NovaMigrationTarget + - OS::TripleO::Services::ContainersLogrotateCrond + - OS::TripleO::Services::Podman + - OS::TripleO::Services::Rhsm + - OS::TripleO::Services::Rsyslog + - OS::TripleO::Services::RsyslogSidecar + - OS::TripleO::Services::Securetty + - OS::TripleO::Services::Snmp + - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Timesync + - OS::TripleO::Services::Timezone + - OS::TripleO::Services::TripleoFirewall + - OS::TripleO::Services::TripleoPackages + - OS::TripleO::Services::Tuned + - OS::TripleO::Services::Vpp + - OS::TripleO::Services::OVNController + - OS::TripleO::Services::OVNMetadataAgent diff --git a/scenarios/dcn_storage/central/vips_data.yaml b/scenarios/dcn_storage/central/vips_data.yaml new file mode 100644 index 000000000..3aff3a399 --- /dev/null +++ b/scenarios/dcn_storage/central/vips_data.yaml @@ -0,0 +1,22 @@ +--- +- name: internal_api_vip + network: internal_api + subnet: internal_api_subnet + dns_name: overcloud +- name: storage_vip + network: storage + subnet: storage_subnet + dns_name: overcloud +- name: storage_mgmt_vip + network: storage_mgmt + subnet: storage_mgmt_subnet + dns_name: overcloud +- name: external_vip + network: external + subnet: external_subnet + dns_name: overcloud +- name: ctlplane_vip + network: ctlplane + ip_address: 192.168.122.99 + subnet: ctlplane-subnet + dns_name: overcloud diff --git a/scenarios/dcn_storage/ceph/osd_spec.yaml b/scenarios/dcn_storage/ceph/osd_spec.yaml new file mode 100644 index 000000000..0433ffab1 --- /dev/null +++ b/scenarios/dcn_storage/ceph/osd_spec.yaml @@ -0,0 +1,3 @@ +--- +data_devices: + all: true diff --git a/scenarios/dcn_storage/dcn1/config_download.yaml b/scenarios/dcn_storage/dcn1/config_download.yaml new file mode 100644 index 000000000..afd30c309 --- /dev/null +++ b/scenarios/dcn_storage/dcn1/config_download.yaml @@ -0,0 +1,126 @@ +--- +resource_registry: + # yamllint disable rule:line-length + OS::TripleO::DeployedServer::ControlPlanePort: /usr/share/openstack-tripleo-heat-templates/deployed-server/deployed-neutron-port.yaml + OS::TripleO::OVNMacAddressNetwork: OS::Heat::None + OS::TripleO::OVNMacAddressPort: OS::Heat::None + OS::TripleO::DistributedComputeHCI::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml + OS::TripleO::DistributedComputeHCI::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml + OS::TripleO::DistributedComputeHCI::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml + OS::TripleO::DistributedComputeHCI::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml + OS::TripleO::Controller::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml + OS::TripleO::Controller::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml + OS::TripleO::Controller::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml + OS::TripleO::Controller::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml + OS::TripleO::Controller::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml + OS::TripleO::Services::CeilometerAgentCentral: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-central-container-puppet.yaml + OS::TripleO::Services::CeilometerAgentNotification: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-notification-container-puppet.yaml + OS::TripleO::Services::CeilometerAgentIpmi: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-ipmi-container-puppet.yaml + OS::TripleO::Services::ComputeCeilometerAgent: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-compute-container-puppet.yaml + OS::TripleO::Services::Collectd: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/collectd-container-puppet.yaml + OS::TripleO::Services::MetricsQdr: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/qdr-container-puppet.yaml + OS::TripleO::Services::OsloMessagingRpc: /usr/share/openstack-tripleo-heat-templates/deployment/rabbitmq/rabbitmq-messaging-rpc-pacemaker-puppet.yaml + OS::TripleO::Services::OsloMessagingNotify: /usr/share/openstack-tripleo-heat-templates/deployment/rabbitmq/rabbitmq-messaging-notify-shared-puppet.yaml + OS::TripleO::Services::HAproxy: /usr/share/openstack-tripleo-heat-templates/deployment/haproxy/haproxy-pacemaker-puppet.yaml + OS::TripleO::Services::Pacemaker: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/pacemaker-baremetal-puppet.yaml + OS::TripleO::Services::PacemakerRemote: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/pacemaker-remote-baremetal-puppet.yaml + OS::TripleO::Services::Clustercheck: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/clustercheck-container-puppet.yaml + OS::TripleO::Services::Redis: /usr/share/openstack-tripleo-heat-templates/deployment/database/redis-pacemaker-puppet.yaml + OS::TripleO::Services::Rsyslog: /usr/share/openstack-tripleo-heat-templates/deployment/logging/rsyslog-container-puppet.yaml + OS::TripleO::Services::MySQL: /usr/share/openstack-tripleo-heat-templates/deployment/database/mysql-pacemaker-puppet.yaml + OS::TripleO::Services::CinderBackup: /usr/share/openstack-tripleo-heat-templates/deployment/cinder/cinder-backup-pacemaker-puppet.yaml + OS::TripleO::Services::CinderVolume: /usr/share/openstack-tripleo-heat-templates/deployment/cinder/cinder-volume-pacemaker-puppet.yaml + OS::TripleO::Services::HeatApi: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-container-puppet.yaml + OS::TripleO::Services::HeatApiCfn: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-cfn-container-puppet.yaml + OS::TripleO::Services::HeatApiCloudwatch: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-cloudwatch-disabled-puppet.yaml + OS::TripleO::Services::HeatEngine: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-engine-container-puppet.yaml + OS::TripleO::Services::NovaAZConfig: /usr/share/openstack-tripleo-heat-templates/deployment/nova/nova-az-config.yaml +parameter_defaults: + RedisVirtualFixedIPs: + - ip_address: 192.168.122.110 + use_neutron: false + OVNDBsVirtualFixedIPs: + - ip_address: 192.168.122.111 + use_neutron: false + ControllerExtraConfig: + nova::compute::libvirt::services::libvirt_virt_type: qemu + nova::compute::libvirt::virt_type: qemu + nova::availability_zone::default_schedule_zone: az-central + DistributedComputeHCIExtraConfig: + nova::compute::libvirt::services::libvirt_virt_type: qemu + nova::compute::libvirt::virt_type: qemu + BarbicanSimpleCryptoGlobalDefault: true + Debug: true + DockerPuppetDebug: true + ContainerCli: podman + DistributedComputeHCICount: 3 + NeutronGlobalPhysnetMtu: 1350 + CinderLVMLoopDeviceSize: 20480 + CloudName: overcloud.example.com + CloudNameInternal: overcloud.internalapi.example.com + CloudNameStorage: overcloud.storage.example.com + CloudNameStorageManagement: overcloud.storagemgmt.example.com + CloudNameCtlplane: overcloud.ctlplane.example.com + CloudDomain: example.com + NetworkConfigWithAnsible: false + ControllerHostnameFormat: '%stackname%-controller-%index%' + DistributedComputeHCIHostnameFormat: '%stackname%-distributedcomputehci-%index%' + DistributedComputeHCIControlPlaneSubnet: leaf1 + CtlplaneNetworkAttributes: + network: + dns_domain: example.com + mtu: 1500 + name: ctlplane + tags: + - 192.168.122.0/24 + - 192.168.133.0/24 + - 192.168.144.0/24 + subnets: + ctlplane-subnet: + cidr: 192.168.122.0/24 + dns_nameservers: 192.168.122.10 + gateway_ip: 192.168.122.10 + host_routes: [] + name: ctlplane-subnet + ip_version: 4 + leaf1: + cidr: 192.168.133.0/24 + dns_nameservers: 192.168.122.10 + gateway_ip: 192.168.133.1 + host_routes: + - destination: 192.168.122.0/24 # Route to central ctlplane for DNS + nexthop: 192.168.133.1 + - destination: 192.168.144.0/24 # Route to DCN2 + nexthop: 192.168.133.1 + name: leaf1 + ip_version: 4 + leaf2: + cidr: 192.168.144.0/24 + dns_nameservers: 192.168.122.10 + gateway_ip: 192.168.144.1 + host_routes: [] + name: leaf2 + ip_version: 4 + + VipSubnetMap: + ctlplane: leaf1 + NeutronFlatNetworks: leaf0,leaf1,leaf2 + ControllerParameters: + NeutronBridgeMappings: ["leaf0:br-ex"] + DistributedComputeHCIParameters: + NeutronBridgeMappings: ["leaf1:br-ex"] + PythonInterpreter: /usr/bin/python3 + DnsServers: ['10.11.5.19', '10.5.30.160'] + NtpServer: ["clock.redhat.com", "clock2.redhat.com"] + NovaComputeAvailabilityZone: az-dcn1 + CinderStorageAvailabilityZone: az-dcn1 + CinderVolumeCluster: dcn1 + NovaCrossAZAttach: false + RootStackName: dcn1 + AuthCloudName: 'central' + ManageNetworks: false + + NeutronNetworkType: geneve + NeutronTunnelTypes: geneve + NeutronNetworkVLANRanges: 'leaf0:1:1000,leaf1:1:1000,leaf2:1:1000' + BondInterfaceOvsOptions: "bond_mode=active-backup" diff --git a/scenarios/dcn_storage/dcn1/glance_dcn1.yaml b/scenarios/dcn_storage/dcn1/glance_dcn1.yaml new file mode 100644 index 000000000..acc65314a --- /dev/null +++ b/scenarios/dcn_storage/dcn1/glance_dcn1.yaml @@ -0,0 +1,11 @@ +parameter_defaults: + GlanceShowMultipleLocations: true + GlanceEnabledImportMethods: web-download,copy-image + GlanceBackend: rbd + GlanceBackendID: dcn1 + GlanceStoreDescription: 'dcn1 rbd glance store' + GlanceMultistoreConfig: + central: + GlanceBackend: rbd + GlanceStoreDescription: 'central rbd glance store' + CephClusterName: central diff --git a/scenarios/dcn_storage/dcn1/network_data.yaml.j2 b/scenarios/dcn_storage/dcn1/network_data.yaml.j2 new file mode 100644 index 000000000..dabe35ded --- /dev/null +++ b/scenarios/dcn_storage/dcn1/network_data.yaml.j2 @@ -0,0 +1,107 @@ +--- +- name: Storage + mtu: 1496 + vip: true + name_lower: storage + dns_domain: storage.{{ cloud_domain }}. + service_net_map_replace: storage + subnets: + storage_subnet: + vlan: 21 + ip_subnet: '172.18.0.0/24' + allocation_pools: [{'start': '172.18.0.120', 'end': '172.18.0.250'}] + storage_leaf1: + vlan: 31 + ip_subnet: '172.18.10.0/24' + allocation_pools: [{'start': '172.18.10.120', 'end': '172.18.10.250'}] + routes: + - destination: 172.18.0.0/24 + nexthop: 172.18.10.1 + - destination: 172.18.20.0/24 + nexthop: 172.18.10.1 + storage_leaf2: + vlan: 41 + ip_subnet: '172.18.20.0/24' + allocation_pools: [{'start': '172.18.20.120', 'end': '172.18.20.250'}] + +- name: StorageMgmt + mtu: 1496 + vip: true + name_lower: storage_mgmt + dns_domain: storagemgmt.{{ cloud_domain }}. + service_net_map_replace: storage_mgmt + subnets: + storage_mgmt_subnet: + vlan: 23 + ip_subnet: '172.20.0.0/24' + allocation_pools: [{'start': '172.20.0.120', 'end': '172.20.0.250'}] + storage_mgmt_leaf1: + vlan: 33 + ip_subnet: '172.20.10.0/24' + allocation_pools: [{'start': '172.20.10.120', 'end': '172.20.10.250'}] + storage_mgmt_leaf2: + vlan: 43 + ip_subnet: '172.20.20.0/24' + allocation_pools: [{'start': '172.20.20.120', 'end': '172.20.20.250'}] + +- name: InternalApi + mtu: 1496 + vip: true + name_lower: internal_api + dns_domain: internal-api.{{ cloud_domain }}. + service_net_map_replace: internal_api + subnets: + internal_api_subnet: + vlan: 20 + ip_subnet: '172.17.0.0/24' + allocation_pools: [{'start': '172.17.0.120', 'end': '172.17.0.250'}] + internal_api_leaf1: + vlan: 30 + ip_subnet: '172.17.10.0/24' + allocation_pools: [{'start': '172.17.10.120', 'end': '172.17.10.250'}] + routes: + - destination: 172.17.0.0/24 + nexthop: 172.17.10.1 + - destination: 172.17.20.0/24 + nexthop: 172.17.10.1 + internal_api_leaf2: + vlan: 40 + ip_subnet: '172.17.20.0/24' + allocation_pools: [{'start': '172.17.20.120', 'end': '172.17.20.250'}] + +- name: Tenant + mtu: 1496 + vip: false # Tenant network does not use VIPs + name_lower: tenant + dns_domain: tenant.{{ cloud_domain }}. + service_net_map_replace: tenant + subnets: + tenant_subnet: + vlan: 22 + ip_subnet: '172.19.0.0/24' + allocation_pools: [{'start': '172.19.0.120', 'end': '172.19.0.250'}] + tenant_leaf_1: + vlan: 32 + ip_subnet: '172.19.10.0/24' + allocation_pools: [{'start': '172.19.10.120', 'end': '172.19.10.250'}] + routes: + - destination: 172.19.0.0/24 + nexthop: 172.19.10.1 + - destination: 172.19.20.0/24 + nexthop: 172.19.10.1 + tenant_leaf_2: + vlan: 42 + ip_subnet: '172.19.20.0/24' + allocation_pools: [{'start': '172.19.20.120', 'end': '172.19.20.250'}] + +- name: External + mtu: 1500 + vip: true + name_lower: external + dns_domain: external.{{ cloud_domain }}. + service_net_map_replace: external + subnets: + external_subnet: + vlan: 44 + ip_subnet: '10.0.0.0/24' + allocation_pools: [{'start': '10.0.0.150', 'end': '10.0.0.250'}] diff --git a/scenarios/dcn_storage/dcn1/roles.yaml b/scenarios/dcn_storage/dcn1/roles.yaml new file mode 100644 index 000000000..1f73947b3 --- /dev/null +++ b/scenarios/dcn_storage/dcn1/roles.yaml @@ -0,0 +1,78 @@ +############################################################################### +# Role: DistributedComputeHCI # +############################################################################### +- name: DistributedComputeHCI + description: | + Distributed Compute Node role with Ceph, Cinder volume, and Glance. + tags: + - compute + networks: + InternalApi: + subnet: internal_api_leaf1 + Tenant: + subnet: tenant_leaf1 + Storage: + subnet: storage_leaf1 + StorageMgmt: + subnet: storage_mgmt_leaf1 + RoleParametersDefault: + TunedProfileName: "throughput-performance" + # CephOSD present so serial has to be 1 + update_serial: 1 + ServicesDefault: + - OS::TripleO::Services::Aide + - OS::TripleO::Services::AuditD + - OS::TripleO::Services::BarbicanClient + - OS::TripleO::Services::BootParams + - OS::TripleO::Services::CACerts + - OS::TripleO::Services::CephClient + - OS::TripleO::Services::CephExternal + - OS::TripleO::Services::CephGrafana + - OS::TripleO::Services::CephMds + - OS::TripleO::Services::CephMgr + - OS::TripleO::Services::CephMon + - OS::TripleO::Services::CephRbdMirror + - OS::TripleO::Services::CephRgw + - OS::TripleO::Services::CephOSD + - OS::TripleO::Services::CinderVolumeEdge + - OS::TripleO::Services::Collectd + - OS::TripleO::Services::ComputeCeilometerAgent + - OS::TripleO::Services::ComputeNeutronCorePlugin + - OS::TripleO::Services::ComputeNeutronL3Agent + - OS::TripleO::Services::ComputeNeutronMetadataAgent + - OS::TripleO::Services::ComputeNeutronOvsAgent + - OS::TripleO::Services::Etcd + - OS::TripleO::Services::Frr + - OS::TripleO::Services::GlanceApiEdge + - OS::TripleO::Services::IpaClient + - OS::TripleO::Services::Ipsec + - OS::TripleO::Services::Iscsid + - OS::TripleO::Services::Kernel + - OS::TripleO::Services::LoginDefs + - OS::TripleO::Services::MetricsQdr + - OS::TripleO::Services::Multipathd + - OS::TripleO::Services::MySQLClient + - OS::TripleO::Services::NeutronBgpVpnBagpipe + - OS::TripleO::Services::NeutronLinuxbridgeAgent + - OS::TripleO::Services::NeutronVppAgent + - OS::TripleO::Services::NovaAZConfig + - OS::TripleO::Services::NovaCompute + - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaLibvirtGuests + - OS::TripleO::Services::NovaMigrationTarget + - OS::TripleO::Services::ContainersLogrotateCrond + - OS::TripleO::Services::Podman + - OS::TripleO::Services::Rhsm + - OS::TripleO::Services::Rsyslog + - OS::TripleO::Services::RsyslogSidecar + - OS::TripleO::Services::Securetty + - OS::TripleO::Services::Snmp + - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Timesync + - OS::TripleO::Services::Timezone + - OS::TripleO::Services::TripleoFirewall + - OS::TripleO::Services::TripleoPackages + - OS::TripleO::Services::Tuned + - OS::TripleO::Services::Vpp + - OS::TripleO::Services::OVNController + - OS::TripleO::Services::OVNMetadataAgent diff --git a/scenarios/dcn_storage/dcn1/vips_data.yaml b/scenarios/dcn_storage/dcn1/vips_data.yaml new file mode 100644 index 000000000..e6fc6ace9 --- /dev/null +++ b/scenarios/dcn_storage/dcn1/vips_data.yaml @@ -0,0 +1,13 @@ +--- +- network: ctlplane + dns_name: overcloud + subnet: leaf1 +- network: internal_api + dns_name: overcloud + subnet: internal_api_leaf1 +- network: storage + dns_name: overcloud + subnet: storage_leaf1 +- network: storage_mgmt + dns_name: overcloud + subnet: storage_mgmt_leaf1 diff --git a/scenarios/dcn_storage/dcn2/config_download.yaml b/scenarios/dcn_storage/dcn2/config_download.yaml new file mode 100644 index 000000000..6f56f3a73 --- /dev/null +++ b/scenarios/dcn_storage/dcn2/config_download.yaml @@ -0,0 +1,126 @@ +--- +resource_registry: + # yamllint disable rule:line-length + OS::TripleO::DeployedServer::ControlPlanePort: /usr/share/openstack-tripleo-heat-templates/deployed-server/deployed-neutron-port.yaml + OS::TripleO::OVNMacAddressNetwork: OS::Heat::None + OS::TripleO::OVNMacAddressPort: OS::Heat::None + OS::TripleO::DistributedComputeHCI::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml + OS::TripleO::DistributedComputeHCI::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml + OS::TripleO::DistributedComputeHCI::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml + OS::TripleO::DistributedComputeHCI::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml + OS::TripleO::Controller::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml + OS::TripleO::Controller::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml + OS::TripleO::Controller::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml + OS::TripleO::Controller::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml + OS::TripleO::Controller::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml + OS::TripleO::Services::CeilometerAgentCentral: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-central-container-puppet.yaml + OS::TripleO::Services::CeilometerAgentNotification: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-notification-container-puppet.yaml + OS::TripleO::Services::CeilometerAgentIpmi: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-ipmi-container-puppet.yaml + OS::TripleO::Services::ComputeCeilometerAgent: /usr/share/openstack-tripleo-heat-templates/deployment/ceilometer/ceilometer-agent-compute-container-puppet.yaml + OS::TripleO::Services::Collectd: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/collectd-container-puppet.yaml + OS::TripleO::Services::MetricsQdr: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/qdr-container-puppet.yaml + OS::TripleO::Services::OsloMessagingRpc: /usr/share/openstack-tripleo-heat-templates/deployment/rabbitmq/rabbitmq-messaging-rpc-pacemaker-puppet.yaml + OS::TripleO::Services::OsloMessagingNotify: /usr/share/openstack-tripleo-heat-templates/deployment/rabbitmq/rabbitmq-messaging-notify-shared-puppet.yaml + OS::TripleO::Services::HAproxy: /usr/share/openstack-tripleo-heat-templates/deployment/haproxy/haproxy-pacemaker-puppet.yaml + OS::TripleO::Services::Pacemaker: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/pacemaker-baremetal-puppet.yaml + OS::TripleO::Services::PacemakerRemote: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/pacemaker-remote-baremetal-puppet.yaml + OS::TripleO::Services::Clustercheck: /usr/share/openstack-tripleo-heat-templates/deployment/pacemaker/clustercheck-container-puppet.yaml + OS::TripleO::Services::Redis: /usr/share/openstack-tripleo-heat-templates/deployment/database/redis-pacemaker-puppet.yaml + OS::TripleO::Services::Rsyslog: /usr/share/openstack-tripleo-heat-templates/deployment/logging/rsyslog-container-puppet.yaml + OS::TripleO::Services::MySQL: /usr/share/openstack-tripleo-heat-templates/deployment/database/mysql-pacemaker-puppet.yaml + OS::TripleO::Services::CinderBackup: /usr/share/openstack-tripleo-heat-templates/deployment/cinder/cinder-backup-pacemaker-puppet.yaml + OS::TripleO::Services::CinderVolume: /usr/share/openstack-tripleo-heat-templates/deployment/cinder/cinder-volume-pacemaker-puppet.yaml + OS::TripleO::Services::HeatApi: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-container-puppet.yaml + OS::TripleO::Services::HeatApiCfn: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-cfn-container-puppet.yaml + OS::TripleO::Services::HeatApiCloudwatch: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-api-cloudwatch-disabled-puppet.yaml + OS::TripleO::Services::HeatEngine: /usr/share/openstack-tripleo-heat-templates/deployment/heat/heat-engine-container-puppet.yaml + OS::TripleO::Services::NovaAZConfig: /usr/share/openstack-tripleo-heat-templates/deployment/nova/nova-az-config.yaml +parameter_defaults: + RedisVirtualFixedIPs: + - ip_address: 192.168.122.110 + use_neutron: false + OVNDBsVirtualFixedIPs: + - ip_address: 192.168.122.111 + use_neutron: false + ControllerExtraConfig: + nova::compute::libvirt::services::libvirt_virt_type: qemu + nova::compute::libvirt::virt_type: qemu + nova::availability_zone::default_schedule_zone: az-central + DistributedComputeHCIExtraConfig: + nova::compute::libvirt::services::libvirt_virt_type: qemu + nova::compute::libvirt::virt_type: qemu + BarbicanSimpleCryptoGlobalDefault: true + Debug: true + DockerPuppetDebug: true + ContainerCli: podman + DistributedComputeHCICount: 3 + NeutronGlobalPhysnetMtu: 1350 + CinderLVMLoopDeviceSize: 20480 + CloudName: overcloud.example.com + CloudNameInternal: overcloud.internalapi.example.com + CloudNameStorage: overcloud.storage.example.com + CloudNameStorageManagement: overcloud.storagemgmt.example.com + CloudNameCtlplane: overcloud.ctlplane.example.com + CloudDomain: example.com + NetworkConfigWithAnsible: false + ControllerHostnameFormat: '%stackname%-controller-%index%' + DistributedComputeHCIHostnameFormat: '%stackname%-distributedcomputehci-%index%' + DistributedComputeHCIControlPlaneSubnet: leaf2 + CtlplaneNetworkAttributes: + network: + dns_domain: example.com + mtu: 1500 + name: ctlplane + tags: + - 192.168.122.0/24 + - 192.168.133.0/24 + - 192.168.144.0/24 + subnets: + ctlplane-subnet: + cidr: 192.168.122.0/24 + dns_nameservers: 192.168.122.10 + gateway_ip: 192.168.122.10 + host_routes: [] + name: ctlplane-subnet + ip_version: 4 + leaf1: + cidr: 192.168.133.0/24 + dns_nameservers: 192.168.122.10 + gateway_ip: 192.168.133.1 + host_routes: [] + name: leaf1 + ip_version: 4 + leaf2: + cidr: 192.168.144.0/24 + dns_nameservers: 192.168.122.10 + gateway_ip: 192.168.144.1 + host_routes: + - destination: 192.168.122.0/24 # Route to central ctlplane for DNS + nexthop: 192.168.144.1 + - destination: 192.168.133.0/24 # Route to DCN1 + nexthop: 192.168.144.1 + name: leaf2 + ip_version: 4 + + VipSubnetMap: + ctlplane: leaf2 + NeutronFlatNetworks: leaf0,leaf1,leaf2 + ControllerParameters: + NeutronBridgeMappings: ["leaf0:br-ex"] + DistributedComputeHCIParameters: + NeutronBridgeMappings: ["leaf2:br-ex"] + PythonInterpreter: /usr/bin/python3 + DnsServers: ['10.11.5.19', '10.5.30.160'] + NtpServer: ["clock.redhat.com", "clock2.redhat.com"] + NovaComputeAvailabilityZone: az-dcn2 + CinderStorageAvailabilityZone: az-dcn2 + CinderVolumeCluster: dcn2 + NovaCrossAZAttach: false + RootStackName: dcn2 + AuthCloudName: 'central' + ManageNetworks: false + + NeutronNetworkType: geneve + NeutronTunnelTypes: geneve + NeutronNetworkVLANRanges: 'leaf0:1:1000,leaf1:1:1000,leaf2:1:1000' + BondInterfaceOvsOptions: "bond_mode=active-backup" diff --git a/scenarios/dcn_storage/dcn2/glance_dcn2.yaml b/scenarios/dcn_storage/dcn2/glance_dcn2.yaml new file mode 100644 index 000000000..97a09f99c --- /dev/null +++ b/scenarios/dcn_storage/dcn2/glance_dcn2.yaml @@ -0,0 +1,11 @@ +parameter_defaults: + GlanceShowMultipleLocations: true + GlanceEnabledImportMethods: web-download,copy-image + GlanceBackend: rbd + GlanceBackendID: dcn2 + GlanceStoreDescription: 'dcn2 rbd glance store' + GlanceMultistoreConfig: + central: + GlanceBackend: rbd + GlanceStoreDescription: 'central rbd glance store' + CephClusterName: central diff --git a/scenarios/dcn_storage/dcn2/network_data.yaml.j2 b/scenarios/dcn_storage/dcn2/network_data.yaml.j2 new file mode 100644 index 000000000..8784fd340 --- /dev/null +++ b/scenarios/dcn_storage/dcn2/network_data.yaml.j2 @@ -0,0 +1,107 @@ +--- +- name: Storage + mtu: 1496 + vip: true + name_lower: storage + dns_domain: storage.{{ cloud_domain }}. + service_net_map_replace: storage + subnets: + storage_subnet: + vlan: 21 + ip_subnet: '172.18.0.0/24' + allocation_pools: [{'start': '172.18.0.120', 'end': '172.18.0.250'}] + storage_leaf1: + vlan: 31 + ip_subnet: '172.18.10.0/24' + allocation_pools: [{'start': '172.18.10.120', 'end': '172.18.10.250'}] + storage_leaf2: + vlan: 41 + ip_subnet: '172.18.20.0/24' + allocation_pools: [{'start': '172.18.20.120', 'end': '172.18.20.250'}] + routes: + - destination: 172.18.0.0/24 + nexthop: 172.18.20.1 + - destination: 172.18.10.0/24 + nexthop: 172.18.20.1 + +- name: StorageMgmt + mtu: 1496 + vip: true + name_lower: storage_mgmt + dns_domain: storagemgmt.{{ cloud_domain }}. + service_net_map_replace: storage_mgmt + subnets: + storage_mgmt_subnet: + vlan: 23 + ip_subnet: '172.20.0.0/24' + allocation_pools: [{'start': '172.20.0.120', 'end': '172.20.0.250'}] + storage_mgmt_leaf1: + vlan: 33 + ip_subnet: '172.20.10.0/24' + allocation_pools: [{'start': '172.20.10.120', 'end': '172.20.10.250'}] + storage_mgmt_leaf2: + vlan: 43 + ip_subnet: '172.20.20.0/24' + allocation_pools: [{'start': '172.20.20.120', 'end': '172.20.20.250'}] + +- name: InternalApi + mtu: 1496 + vip: true + name_lower: internal_api + dns_domain: internal-api.{{ cloud_domain }}. + service_net_map_replace: internal_api + subnets: + internal_api_subnet: + vlan: 20 + ip_subnet: '172.17.0.0/24' + allocation_pools: [{'start': '172.17.0.120', 'end': '172.17.0.250'}] + internal_api_leaf1: + vlan: 30 + ip_subnet: '172.17.10.0/24' + allocation_pools: [{'start': '172.17.10.120', 'end': '172.17.10.250'}] + internal_api_leaf2: + vlan: 40 + ip_subnet: '172.17.20.0/24' + allocation_pools: [{'start': '172.17.20.120', 'end': '172.17.20.250'}] + routes: + - destination: 172.17.0.0/24 + nexthop: 172.17.20.1 + - destination: 172.17.10.0/24 + nexthop: 172.17.20.1 + +- name: Tenant + mtu: 1496 + vip: false # Tenant network does not use VIPs + name_lower: tenant + dns_domain: tenant.{{ cloud_domain }}. + service_net_map_replace: tenant + subnets: + tenant_subnet: + vlan: 22 + ip_subnet: '172.19.0.0/24' + allocation_pools: [{'start': '172.19.0.120', 'end': '172.19.0.250'}] + tenant_leaf_1: + vlan: 32 + ip_subnet: '172.19.10.0/24' + allocation_pools: [{'start': '172.19.10.120', 'end': '172.19.10.250'}] + tenant_leaf_2: + vlan: 42 + ip_subnet: '172.19.20.0/24' + allocation_pools: [{'start': '172.19.20.120', 'end': '172.19.20.250'}] + routes: + - destination: 172.19.0.0/24 + nexthop: 172.19.20.1 + - destination: 172.19.10.0/24 + nexthop: 172.19.20.1 + +- name: External + mtu: 1500 + vip: true + name_lower: external + dns_domain: external.{{ cloud_domain }}. + service_net_map_replace: external + subnets: + external_subnet: + vlan: 44 + ip_subnet: '10.0.0.0/24' + allocation_pools: [{'start': '10.0.0.150', 'end': '10.0.0.250'}] diff --git a/scenarios/dcn_storage/dcn2/roles.yaml b/scenarios/dcn_storage/dcn2/roles.yaml new file mode 100644 index 000000000..beae70317 --- /dev/null +++ b/scenarios/dcn_storage/dcn2/roles.yaml @@ -0,0 +1,78 @@ +############################################################################### +# Role: DistributedComputeHCI # +############################################################################### +- name: DistributedComputeHCI + description: | + Distributed Compute Node role with Ceph, Cinder volume, and Glance. + tags: + - compute + networks: + InternalApi: + subnet: internal_api_leaf2 + Tenant: + subnet: tenant_leaf2 + Storage: + subnet: storage_leaf2 + StorageMgmt: + subnet: storage_mgmt_leaf2 + RoleParametersDefault: + TunedProfileName: "throughput-performance" + # CephOSD present so serial has to be 1 + update_serial: 1 + ServicesDefault: + - OS::TripleO::Services::Aide + - OS::TripleO::Services::AuditD + - OS::TripleO::Services::BarbicanClient + - OS::TripleO::Services::BootParams + - OS::TripleO::Services::CACerts + - OS::TripleO::Services::CephClient + - OS::TripleO::Services::CephExternal + - OS::TripleO::Services::CephGrafana + - OS::TripleO::Services::CephMds + - OS::TripleO::Services::CephMgr + - OS::TripleO::Services::CephMon + - OS::TripleO::Services::CephRbdMirror + - OS::TripleO::Services::CephRgw + - OS::TripleO::Services::CephOSD + - OS::TripleO::Services::CinderVolumeEdge + - OS::TripleO::Services::Collectd + - OS::TripleO::Services::ComputeCeilometerAgent + - OS::TripleO::Services::ComputeNeutronCorePlugin + - OS::TripleO::Services::ComputeNeutronL3Agent + - OS::TripleO::Services::ComputeNeutronMetadataAgent + - OS::TripleO::Services::ComputeNeutronOvsAgent + - OS::TripleO::Services::Etcd + - OS::TripleO::Services::Frr + - OS::TripleO::Services::GlanceApiEdge + - OS::TripleO::Services::IpaClient + - OS::TripleO::Services::Ipsec + - OS::TripleO::Services::Iscsid + - OS::TripleO::Services::Kernel + - OS::TripleO::Services::LoginDefs + - OS::TripleO::Services::MetricsQdr + - OS::TripleO::Services::Multipathd + - OS::TripleO::Services::MySQLClient + - OS::TripleO::Services::NeutronBgpVpnBagpipe + - OS::TripleO::Services::NeutronLinuxbridgeAgent + - OS::TripleO::Services::NeutronVppAgent + - OS::TripleO::Services::NovaAZConfig + - OS::TripleO::Services::NovaCompute + - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaLibvirtGuests + - OS::TripleO::Services::NovaMigrationTarget + - OS::TripleO::Services::ContainersLogrotateCrond + - OS::TripleO::Services::Podman + - OS::TripleO::Services::Rhsm + - OS::TripleO::Services::Rsyslog + - OS::TripleO::Services::RsyslogSidecar + - OS::TripleO::Services::Securetty + - OS::TripleO::Services::Snmp + - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Timesync + - OS::TripleO::Services::Timezone + - OS::TripleO::Services::TripleoFirewall + - OS::TripleO::Services::TripleoPackages + - OS::TripleO::Services::Tuned + - OS::TripleO::Services::Vpp + - OS::TripleO::Services::OVNController + - OS::TripleO::Services::OVNMetadataAgent diff --git a/scenarios/dcn_storage/dcn2/vips_data.yaml b/scenarios/dcn_storage/dcn2/vips_data.yaml new file mode 100644 index 000000000..9de5ce2d0 --- /dev/null +++ b/scenarios/dcn_storage/dcn2/vips_data.yaml @@ -0,0 +1,13 @@ +--- +- network: ctlplane + dns_name: overcloud + subnet: leaf2 +- network: internal_api + dns_name: overcloud + subnet: internal_api_leaf2 +- network: storage + dns_name: overcloud + subnet: storage_leaf2 +- network: storage_mgmt + dns_name: overcloud + subnet: storage_mgmt_leaf2 diff --git a/scenarios/dcn_storage/hieradata_overrides_undercloud.yaml b/scenarios/dcn_storage/hieradata_overrides_undercloud.yaml new file mode 100644 index 000000000..7f191a27f --- /dev/null +++ b/scenarios/dcn_storage/hieradata_overrides_undercloud.yaml @@ -0,0 +1,3 @@ +--- +parameter_defaults: + UndercloudExtraConfig: {} diff --git a/scenarios/dcn_storage/undercloud_parameter_defaults.yaml b/scenarios/dcn_storage/undercloud_parameter_defaults.yaml new file mode 100644 index 000000000..64e2481da --- /dev/null +++ b/scenarios/dcn_storage/undercloud_parameter_defaults.yaml @@ -0,0 +1,14 @@ +--- +{ + "parameter_defaults": { + "MasqueradeNetworks": { + "10.0.0.1/24": [ + "10.0.0.1/24" + ], + "192.168.122.0/24": [ + "192.168.122.0/24" + ] + } + }, + "resource_registry": {} +} diff --git a/tests/roles/ceph_backend_configuration/defaults/main.yaml b/tests/roles/ceph_backend_configuration/defaults/main.yaml index f7669e0b1..070465d89 100644 --- a/tests/roles/ceph_backend_configuration/defaults/main.yaml +++ b/tests/roles/ceph_backend_configuration/defaults/main.yaml @@ -1,4 +1,20 @@ -# rhoso namespace +# Ceph backend configuration defaults + +# Single cluster (backwards compatible default) +# For multi-cluster DCN, override with ceph_clusters list in vars file +ceph_clusters: [] + +# Path where ceph configs are stored on the controller. +# Default is the TripleO path; override with /etc/ceph if CephConfigPath was customized. +ceph_config_dir: /var/lib/tripleo-config/ceph + +# Example for DCN with multiple Ceph clusters: +# ceph_clusters: +# - name: central # central.conf, central.client.openstack.keyring +# - name: dcn1 # dcn1.conf, dcn1.client.openstack.keyring +# - name: dcn2 # dcn2.conf, dcn2.client.openstack.keyring + +# Control plane patch for extraMounts - mounts all ceph configs to /etc/ceph ceph_backend_configuration_patch: | spec: extraMounts: diff --git a/tests/roles/ceph_backend_configuration/tasks/main.yaml b/tests/roles/ceph_backend_configuration/tasks/main.yaml index 041fdf7dd..415e04bfc 100644 --- a/tests/roles/ceph_backend_configuration/tasks/main.yaml +++ b/tests/roles/ceph_backend_configuration/tasks/main.yaml @@ -1,4 +1,9 @@ -- name: set shell vars for stopping openstack services +# Ceph backend configuration for OpenStack adoption +# Supports single cluster (backwards compatible) or multiple clusters (DCN) + +# Backwards compatible: single cluster mode when ceph_clusters is empty +- name: Single cluster - set shell vars for ceph configuration + when: ceph_clusters | length == 0 no_log: "{{ use_no_log }}" ansible.builtin.set_fact: ceph_backend_configuration_shell_vars: | @@ -8,7 +13,8 @@ CEPH_KEY=$($CEPH_SSH "cat /etc/ceph/ceph.client.openstack.keyring | base64 -w 0") CEPH_CONF=$($CEPH_SSH "cat /etc/ceph/ceph.conf | base64 -w 0") -- name: update the openstack keyring caps for Manila +- name: Single cluster - update the openstack keyring caps for Manila + when: ceph_clusters | length == 0 no_log: "{{ use_no_log }}" ansible.builtin.shell: | {{ shell_header }} @@ -18,7 +24,8 @@ OSP_KEYRING="client.openstack" CEPH_ADM=$($CEPH_SSH "cephadm shell -- ceph auth caps $OSP_KEYRING $CEPH_CAPS") -- name: create ceph-conf-files secret +- name: Single cluster - create ceph-conf-files secret + when: ceph_clusters | length == 0 no_log: "{{ use_no_log }}" ansible.builtin.shell: | {{ shell_header }} @@ -33,12 +40,113 @@ kind: Secret metadata: name: ceph-conf-files + namespace: {{ rhoso_namespace }} type: Opaque EOF -- name: configure control plane extraMounts +# Multi-cluster mode for DCN +# All ceph configs are already on the controller at /var/lib/tripleo-config/ceph/ +- name: Multi-cluster - update openstack keyring caps for Manila on central cluster + when: ceph_clusters | length > 0 + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + CEPH_SSH="{{ controller1_ssh }}" + CEPH_CAPS="mgr 'allow *' mon 'allow r, profile rbd' osd 'profile rbd pool=vms, profile rbd pool=volumes, profile rbd pool=images, profile rbd pool=backups, allow rw pool manila_data'" + $CEPH_SSH "sudo cephadm shell --config /etc/ceph/central.conf --keyring /etc/ceph/central.client.admin.keyring -- ceph auth caps client.openstack $CEPH_CAPS" + +- name: Multi-cluster - copy ceph config files from controller to local temp dir + when: ceph_clusters | length > 0 + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + CEPH_SSH="{{ controller1_ssh }}" + CEPH_DIR="{{ ceph_config_dir }}" + TMPDIR=$(mktemp -d) + {% for cluster in ceph_clusters %} + $CEPH_SSH "cat ${CEPH_DIR}/{{ cluster.name }}.conf" > ${TMPDIR}/{{ cluster.name }}.conf + $CEPH_SSH "sudo cat ${CEPH_DIR}/{{ cluster.name }}.client.openstack.keyring" > ${TMPDIR}/{{ cluster.name }}.client.openstack.keyring + {% endfor %} + echo "tmpdir=${TMPDIR}" + register: ceph_tmpdir_result + +- name: Multi-cluster - extract fsids from copied conf files + when: ceph_clusters | length > 0 + ansible.builtin.shell: | + set -o pipefail + TMPDIR=$(echo "{{ ceph_tmpdir_result.stdout_lines | join('\n') }}" | grep '^tmpdir=' | sed 's/tmpdir=//') + {% for cluster in ceph_clusters %} + FSID_{{ cluster.name }}=$(awk '/fsid/{print $3}' ${TMPDIR}/{{ cluster.name }}.conf) + echo "{{ cluster.name }}=${FSID_{{ cluster.name }}}" + {% endfor %} + register: ceph_fsid_result + +- name: Multi-cluster - build fsid map from extracted fsids + when: ceph_clusters | length > 0 + ansible.builtin.set_fact: + ceph_cluster_fsids: >- + {%- set fsids = {} -%} + {%- for line in ceph_fsid_result.stdout_lines -%} + {%- set parts = line.split('=', 1) -%} + {%- if parts | length == 2 -%} + {%- set _ = fsids.update({parts[0]: parts[1]}) -%} + {%- endif -%} + {%- endfor -%} + {{ fsids }} + +- name: Multi-cluster - create per-site ceph secrets + when: ceph_clusters | length > 0 + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + TMPDIR=$(echo "{{ ceph_tmpdir_result.stdout_lines | join('\n') }}" | grep '^tmpdir=' | sed 's/tmpdir=//') + + # Central site (first cluster) gets all keys + oc delete secret ceph-conf-{{ ceph_clusters[0].name }} -n {{ rhoso_namespace }} --ignore-not-found + oc create secret generic ceph-conf-{{ ceph_clusters[0].name }} \ + {% for cluster in ceph_clusters %} + --from-file=${TMPDIR}/{{ cluster.name }}.conf \ + --from-file=${TMPDIR}/{{ cluster.name }}.client.openstack.keyring \ + {% endfor %} + -n {{ rhoso_namespace }} + + # Each edge site gets central + local keys only + {% for cluster in ceph_clusters[1:] %} + oc delete secret ceph-conf-{{ cluster.name }} -n {{ rhoso_namespace }} --ignore-not-found + oc create secret generic ceph-conf-{{ cluster.name }} \ + --from-file=${TMPDIR}/{{ ceph_clusters[0].name }}.conf \ + --from-file=${TMPDIR}/{{ ceph_clusters[0].name }}.client.openstack.keyring \ + --from-file=${TMPDIR}/{{ cluster.name }}.conf \ + --from-file=${TMPDIR}/{{ cluster.name }}.client.openstack.keyring \ + -n {{ rhoso_namespace }} + {% endfor %} + + rm -rf ${TMPDIR} + +# Single cluster: configure control plane extraMounts with a single secret +- name: Configure control plane extraMounts + when: ceph_clusters | length == 0 ansible.builtin.shell: | {{ shell_header }} {{ oc_header }} oc patch openstackcontrolplane openstack --type=merge --patch '{{ ceph_backend_configuration_patch }}' + +# Multi-cluster (DCN): configure control plane extraMounts with per-site secrets and propagation +- name: Render control plane extraMounts patch for DCN + when: ceph_clusters | length > 0 + ansible.builtin.template: + src: ceph_dcn_extramounts.yaml.j2 + dest: /tmp/ceph-dcn-extramounts.yaml + mode: '0644' + +- name: Configure control plane extraMounts for DCN + when: ceph_clusters | length > 0 + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + + oc patch openstackcontrolplane openstack --type=merge --patch-file /tmp/ceph-dcn-extramounts.yaml + rm -f /tmp/ceph-dcn-extramounts.yaml diff --git a/tests/roles/ceph_backend_configuration/templates/ceph_dcn_extramounts.yaml.j2 b/tests/roles/ceph_backend_configuration/templates/ceph_dcn_extramounts.yaml.j2 new file mode 100644 index 000000000..d91227629 --- /dev/null +++ b/tests/roles/ceph_backend_configuration/templates/ceph_dcn_extramounts.yaml.j2 @@ -0,0 +1,35 @@ +spec: + extraMounts: + - name: v1 + region: r1 + extraVol: + - extraVolType: Ceph + propagation: + - {{ ceph_clusters[0].name }} + - CinderBackup + - ManilaShare + volumes: + - name: ceph-{{ ceph_clusters[0].name }} + projected: + sources: + - secret: + name: ceph-conf-{{ ceph_clusters[0].name }} + mounts: + - name: ceph-{{ ceph_clusters[0].name }} + mountPath: "/etc/ceph" + readOnly: true +{% for cluster in ceph_clusters[1:] %} + - extraVolType: Ceph + propagation: + - {{ cluster.name }} + volumes: + - name: ceph-{{ cluster.name }} + projected: + sources: + - secret: + name: ceph-conf-{{ cluster.name }} + mounts: + - name: ceph-{{ cluster.name }} + mountPath: "/etc/ceph" + readOnly: true +{% endfor %} diff --git a/tests/roles/cinder_adoption/defaults/main.yaml b/tests/roles/cinder_adoption/defaults/main.yaml index 57d2b9782..f18dbe0e4 100644 --- a/tests/roles/cinder_adoption/defaults/main.yaml +++ b/tests/roles/cinder_adoption/defaults/main.yaml @@ -1,8 +1,25 @@ # Volume backends: ceph, ontap-nfs, ontap-iscsi, pure-fc +# For DCN multi-backend Ceph, set cinder_volume_backend: ceph and populate cinder_dcn_clusters # Backup backends: ceph, swift, ontap-nfs cinder_volume_backend: "" cinder_backup_backend: "" +# DCN multi-backend configuration +# Each cluster entry creates a separate cinderVolume with its own AZ +cinder_dcn_clusters: [] +# Example: +# - name: central +# ceph_conf: /etc/ceph/central.conf +# rbd_pool: volumes +# availability_zone: central +# rbd_secret_uuid: # optional, for libvirt +# - name: dcn1 +# ceph_conf: /etc/ceph/dcn1.conf +# rbd_pool: volumes +# availability_zone: dcn1 +# rbd_secret_uuid: +cinder_dcn_default_az: central + cinder_api_patch: | spec: cinder: diff --git a/tests/roles/cinder_adoption/tasks/cinder_volume_dcn.yaml b/tests/roles/cinder_adoption/tasks/cinder_volume_dcn.yaml new file mode 100644 index 000000000..2774ae5ba --- /dev/null +++ b/tests/roles/cinder_adoption/tasks/cinder_volume_dcn.yaml @@ -0,0 +1,11 @@ +- name: Generate Cinder DCN patch file + ansible.builtin.template: + src: cinder_dcn_patch.yaml.j2 + dest: /tmp/cinder_dcn_patch.yaml + mode: '0644' + +- name: Deploy podified Cinder volume with DCN multi-backend configuration + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc patch openstackcontrolplane openstack --type=merge --patch-file /tmp/cinder_dcn_patch.yaml diff --git a/tests/roles/cinder_adoption/tasks/volume_backend.yaml b/tests/roles/cinder_adoption/tasks/volume_backend.yaml index 3a11342ad..c2d46f3bb 100644 --- a/tests/roles/cinder_adoption/tasks/volume_backend.yaml +++ b/tests/roles/cinder_adoption/tasks/volume_backend.yaml @@ -1,10 +1,18 @@ - name: deploy podified Cinder volume - when: cinder_volume_backend == 'ceph' + when: + - cinder_volume_backend == 'ceph' + - cinder_dcn_clusters | length == 0 ansible.builtin.shell: | {{ shell_header }} {{ oc_header }} oc patch openstackcontrolplane openstack --type=merge --patch '{{ cinder_volume_backend_patch }}' +- name: deploy podified Cinder volume with Ceph multi-backend configuration for DCN + when: + - cinder_volume_backend == 'ceph' + - cinder_dcn_clusters | length > 0 + ansible.builtin.include_tasks: cinder_volume_dcn.yaml + - name: deploy podified Cinder volume with netapp NFS or iSCSI when: > cinder_volume_backend == 'ontap-nfs' or diff --git a/tests/roles/cinder_adoption/templates/cinder_dcn_patch.yaml.j2 b/tests/roles/cinder_adoption/templates/cinder_dcn_patch.yaml.j2 new file mode 100644 index 000000000..e2445c416 --- /dev/null +++ b/tests/roles/cinder_adoption/templates/cinder_dcn_patch.yaml.j2 @@ -0,0 +1,39 @@ +spec: + cinder: + enabled: true + template: + cinderAPI: + customServiceConfig: | + [DEFAULT] + default_availability_zone = {{ cinder_dcn_default_az | default('central') }} + cinderVolumes: +{% for cluster in cinder_dcn_clusters %} + {{ cluster.name }}: + networkAttachments: + - storage + replicas: 1 + customServiceConfig: | + [DEFAULT] + enabled_backends = {{ cluster.name }} +{% if cluster.glance_api is defined %} + glance_api_servers = {{ cluster.glance_api }} +{% else %} + glance_api_servers = http://glance-{{ cluster.name }}-internal.openstack.svc:9292 +{% endif %} + [{{ cluster.name }}] + backend_host = hostgroup + volume_backend_name = {{ cluster.name }} + volume_driver = cinder.volume.drivers.rbd.RBDDriver + rbd_ceph_conf = {{ cluster.ceph_conf | default('/etc/ceph/' + cluster.name + '.conf') }} + rbd_user = {{ cluster.rbd_user | default('openstack') }} + rbd_pool = {{ cluster.rbd_pool | default('volumes') }} + rbd_flatten_volume_from_snapshot = False + report_discard_supported = True +{% if cluster.rbd_secret_uuid is defined %} + rbd_secret_uuid = {{ cluster.rbd_secret_uuid }} +{% elif ceph_cluster_fsids is defined and cluster.name in ceph_cluster_fsids %} + rbd_secret_uuid = {{ ceph_cluster_fsids[cluster.name] }} +{% endif %} + rbd_cluster_name = {{ cluster.name }} + backend_availability_zone = {{ cluster.availability_zone | default(cluster.name) }} +{% endfor %} diff --git a/tests/roles/dataplane_adoption/defaults/main.yaml b/tests/roles/dataplane_adoption/defaults/main.yaml index 73262ebe1..a46304859 100644 --- a/tests/roles/dataplane_adoption/defaults/main.yaml +++ b/tests/roles/dataplane_adoption/defaults/main.yaml @@ -224,6 +224,11 @@ edpm_sshd_allowed_ranges: > edpm_neutron_sriov_agent_enabled: true edpm_neutron_dhcp_agent_enabled: true nova_libvirt_backend: local +# Key name in ceph-conf-files secret for extracting CEPH_FSID +# Single cluster: "ceph.conf", DCN multi-cluster: "central.conf" +ceph_conf_secret_key: "ceph.conf" +# DCN per-site Nova Ceph configuration (empty = not DCN) +nova_dcn_clusters: [] skip_patching_ansibleee_csv: false # OS Diff automation steps os_diff_dir: tmp/os-diff diff --git a/tests/roles/dataplane_adoption/tasks/main.yaml b/tests/roles/dataplane_adoption/tasks/main.yaml index ddc5dece7..0b383ecf1 100644 --- a/tests/roles/dataplane_adoption/tasks/main.yaml +++ b/tests/roles/dataplane_adoption/tasks/main.yaml @@ -4,7 +4,7 @@ - ('ceph' in [nova_libvirt_backend]) ansible.builtin.set_fact: ceph_backend_configuration_fsid_shell_vars: | - CEPH_FSID=$(oc get secret ceph-conf-files -o json | jq -r '.data."ceph.conf"' | base64 -d | grep fsid | sed -e 's/fsid = //') + CEPH_FSID=$(oc get secret ceph-conf-files -o json | jq -r '.data."{{ ceph_conf_secret_key }}"' | base64 -d | grep fsid | sed -e 's/fsid = //') # FIXME: missing docs coverage? - name: Patch openstackversion to use image built from source or latest if none is defined @@ -202,6 +202,7 @@ when: - compute_adoption|bool - ('ceph' in [nova_libvirt_backend]) + - nova_dcn_clusters | length == 0 no_log: "{{ use_no_log }}" ansible.builtin.shell: | {{ shell_header }} @@ -228,9 +229,84 @@ rbd_secret_uuid=$CEPH_FSID EOF +- name: create per-site Nova Ceph ConfigMaps for DCN + when: + - compute_adoption|bool + - ('ceph' in [nova_libvirt_backend]) + - nova_dcn_clusters | length > 0 + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {% for cluster in nova_dcn_clusters %} + FSID_{{ cluster.name }}=$(oc get secret ceph-conf-{{ nova_dcn_clusters[0].name }} -o json | jq -r '.data."{{ cluster.name }}.conf"' | base64 -d | awk '/fsid/{print $3}') + {% endfor %} + {% for cluster in nova_dcn_clusters %} + oc apply -f - < 0 + no_log: "{{ use_no_log }}" + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ cells_env }} + {% for cluster in nova_dcn_clusters %} + oc apply -f - < 0 + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ cells_env }} + + for CELL in $(echo $RENAMED_CELLS); do + test -f nodeset-${CELL}.yaml || continue + oc patch osdpns/openstack-$CELL --type=merge --patch " + spec: + services: + - bootstrap + - download-cache + - configure-network + - validate-network + - install-os + - configure-os + - ssh-known-hosts + - run-os + - reboot-os + - install-certs + - ceph-client + - ovn + - neutron-metadata + - libvirt + - nova-custom-ceph-{{ nova_dcn_clusters[0].name }} + {% if telemetry_adoption|bool +%} + - telemetry + {%+ endif +%} + nodeTemplate: + extraMounts: + - extraVolType: Ceph + volumes: + - name: ceph + secret: + secretName: ceph-conf-{{ nova_dcn_clusters[0].name }} + mounts: + - name: ceph + mountPath: "/etc/ceph" + readOnly: true + " + done + +- name: prepare DCN edge nodesets to use per-site Ceph backend + no_log: "{{ use_no_log }}" + when: + - compute_adoption|bool + - nova_dcn_clusters | length > 0 + - edpm_nodes_dcn1 is defined or edpm_nodes_dcn2 is defined + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + + {% for cluster in nova_dcn_clusters[1:] %} + if oc get openstackdataplanenodeset {{ cluster.name }} -n openstack &>/dev/null; then + SERVICES=$(oc get openstackdataplanenodeset {{ cluster.name }} -n openstack -o jsonpath='{.spec.services}' | \ + sed 's/"nova-cell1"/"nova-custom-ceph-{{ cluster.name }}"/g' | \ + sed 's/"install-certs"/"install-certs","ceph-client"/g') + + oc patch openstackdataplanenodeset {{ cluster.name }} -n openstack --type=merge --patch " + spec: + services: $SERVICES + nodeTemplate: + extraMounts: + - extraVolType: Ceph + volumes: + - name: ceph + secret: + secretName: ceph-conf-{{ cluster.name }} + mounts: + - name: ceph + mountPath: "/etc/ceph" + readOnly: true + " + echo "Patched {{ cluster.name }} nodeset to use nova-custom-ceph-{{ cluster.name }} service" + fi + {% endfor %} + - name: enable neutron-sriov-nic-agent in the OpenStackDataPlaneNodeSet CR no_log: "{{ use_no_log }}" ansible.builtin.shell: | diff --git a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml index dedadcbb8..a0516d263 100644 --- a/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml +++ b/tests/roles/dataplane_adoption/tasks/nova_ffu.yaml @@ -68,6 +68,7 @@ tasks_from: wait.yaml - name: remove the pre-fast-forward upgrade workarounds from the Compute data plane services + when: nova_dcn_clusters | length == 0 ansible.builtin.shell: | {{ shell_header }} {{ nodesets_env_oc }} @@ -77,10 +78,6 @@ for CELL in $(echo $RENAMED_CELLS); do oc get Openstackdataplanenodeset openstack-${CELL} || continue NODESETS="[openstack-${CELL}]" - if [ "$CELL" == "cell1" ]; then - oc get Openstackdataplanenodeset dcn1 && NODESETS="${NODESETS%]*},dcn1]" - oc get Openstackdataplanenodeset dcn2 && NODESETS="${NODESETS%]*},dcn2]" - fi oc apply -f - < 0 + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + {{ cells_env }} + + # Patch per-site ConfigMaps to clear the FFU workaround + {% for cluster in nova_dcn_clusters %} + oc patch cm nova-ceph-{{ cluster.name }} --type=json \ + -p='[{"op": "replace", "path": "/data/99-nova-compute-cells-workarounds.conf", "value": "[workarounds]\n"}]' + {% endfor %} + + # Create FFU deployment for the central nodeset + for CELL in $(echo $RENAMED_CELLS); do + oc get Openstackdataplanenodeset openstack-${CELL} || continue + oc apply -f - </dev/null; then + oc apply -f - < 0 + +- name: deploy podified Glance with DCN multi-backend configuration + ansible.builtin.shell: | + {{ shell_header }} + {{ oc_header }} + oc patch openstackcontrolplane openstack --type=merge --patch-file=/tmp/glance_dcn.yaml diff --git a/tests/roles/glance_adoption/tasks/main.yaml b/tests/roles/glance_adoption/tasks/main.yaml index f568c366a..f26b9bee8 100644 --- a/tests/roles/glance_adoption/tasks/main.yaml +++ b/tests/roles/glance_adoption/tasks/main.yaml @@ -3,7 +3,9 @@ ansible.builtin.include_tasks: glance_nfs.yaml - name: Deploy podified Glance with Ceph backend - when: glance_backend == 'ceph' + when: + - glance_backend == 'ceph' + - glance_dcn_clusters | length == 0 ansible.builtin.include_tasks: glance_ceph.yaml - name: Deploy podified Glance with Swift backend @@ -14,6 +16,12 @@ when: glance_backend == 'cinder' ansible.builtin.include_tasks: glance_cinder.yaml +- name: Deploy podified Glance with Ceph multi-backend configuration for DCN + when: + - glance_backend == 'ceph' + - glance_dcn_clusters | length > 0 + ansible.builtin.include_tasks: glance_dcn.yaml + - name: Apply additional Glance settings ansible.builtin.include_tasks: extra_glance_settings.yaml when: glance_nodeselector | length > 0 diff --git a/tests/roles/glance_adoption/templates/glance_dcn.yaml.j2 b/tests/roles/glance_adoption/templates/glance_dcn.yaml.j2 new file mode 100644 index 000000000..77b39a38b --- /dev/null +++ b/tests/roles/glance_adoption/templates/glance_dcn.yaml.j2 @@ -0,0 +1,57 @@ +spec: + glance: + enabled: true + template: + databaseInstance: openstack + databaseAccount: glance + keystoneEndpoint: {{ glance_dcn_clusters | selectattr('default', 'defined') | selectattr('default') | map(attribute='name') | first }} + storage: + storageRequest: 10G + glanceAPIs: +{% for cluster in glance_dcn_clusters %} + {{ cluster.name }}: + type: {{ cluster.type | default('split') }} + replicas: 1 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: {{ cluster.metallb_ip }} + spec: + type: LoadBalancer + networkAttachments: + - storage + customServiceConfig: | + [DEFAULT] + enabled_import_methods = [web-download,copy-image,glance-direct] +{% if cluster.backends == 'all' %} + enabled_backends = {{ glance_dcn_clusters | map(attribute='name') | join(':rbd,') }}:rbd +{% else %} + enabled_backends = {{ cluster.backends | join(':rbd,') }}:rbd +{% endif %} + [glance_store] + default_backend = {{ cluster.name }} +{% if cluster.backends == 'all' %} +{% for backend in glance_dcn_clusters %} + [{{ backend.name }}] + rbd_store_ceph_conf = {{ backend.ceph_conf }} + store_description = "{{ backend.name }} RBD backend" + rbd_store_pool = {{ backend.pool | default('images') }} + rbd_store_user = openstack + rbd_thin_provisioning = True +{% endfor %} +{% else %} +{% for backend_name in cluster.backends %} +{% set backend = glance_dcn_clusters | selectattr('name', 'equalto', backend_name) | first %} + [{{ backend.name }}] + rbd_store_ceph_conf = {{ backend.ceph_conf }} + store_description = "{{ backend.name }} RBD backend" + rbd_store_pool = {{ backend.pool | default('images') }} + rbd_store_user = openstack + rbd_thin_provisioning = True +{% endfor %} +{% endif %} +{% endfor %} diff --git a/tests/roles/manila_adoption/defaults/main.yaml b/tests/roles/manila_adoption/defaults/main.yaml index 115ae7322..ad7b7d432 100644 --- a/tests/roles/manila_adoption/defaults/main.yaml +++ b/tests/roles/manila_adoption/defaults/main.yaml @@ -26,3 +26,11 @@ supported_backends: - cephnfs - netapp manila_retry_delay: 5 +# Path to the Ceph configuration file for the CephFS backend. +# Default is /etc/ceph/ceph.conf (single-cluster deployments). +# Override with the cluster-specific path for DCN deployments, +# e.g. /etc/ceph/central.conf +manila_cephfs_conf_path: /etc/ceph/ceph.conf +# Ceph cluster name for the CephFS backend. Default is 'ceph'. +# Override for DCN deployments, e.g. 'central'. +manila_cephfs_cluster_name: ceph diff --git a/tests/roles/manila_adoption/templates/manila_cephfs.yaml.j2 b/tests/roles/manila_adoption/templates/manila_cephfs.yaml.j2 index 7ec1024ff..910777121 100644 --- a/tests/roles/manila_adoption/templates/manila_cephfs.yaml.j2 +++ b/tests/roles/manila_adoption/templates/manila_cephfs.yaml.j2 @@ -46,9 +46,9 @@ spec: driver_handles_share_servers=False share_backend_name=cephfs share_driver=manila.share.drivers.cephfs.driver.CephFSDriver - cephfs_conf_path=/etc/ceph/ceph.conf + cephfs_conf_path={{ manila_cephfs_conf_path }} cephfs_auth_id=openstack - cephfs_cluster_name=ceph + cephfs_cluster_name={{ manila_cephfs_cluster_name }} cephfs_volume_mode=0755 cephfs_protocol_helper_type= {{ (manila_backend == "cephfs") | ternary('CEPHFS', 'NFS') }} {% if manila_backend == "cephnfs" -%} diff --git a/tests/roles/stop_openstack_services/defaults/main.yaml b/tests/roles/stop_openstack_services/defaults/main.yaml new file mode 100644 index 000000000..1d7899202 --- /dev/null +++ b/tests/roles/stop_openstack_services/defaults/main.yaml @@ -0,0 +1,9 @@ +# DCN storage nodes - list of nodes with ssh connection strings +# Used to stop Glance/Cinder/etcd services on DCN compute nodes +dcn_storage_nodes: [] + +# Services to stop on DCN storage nodes +dcn_storage_services: + - tripleo_glance_api_internal.service + - tripleo_cinder_volume.service + - tripleo_etcd.service diff --git a/tests/roles/stop_openstack_services/tasks/main.yaml b/tests/roles/stop_openstack_services/tasks/main.yaml index 74c8d91b9..b15c90010 100644 --- a/tests/roles/stop_openstack_services/tasks/main.yaml +++ b/tests/roles/stop_openstack_services/tasks/main.yaml @@ -160,3 +160,31 @@ break fi done + +# DCN storage nodes: stop Glance, Cinder, and etcd services +- name: stop services on DCN storage nodes + when: dcn_storage_nodes | length > 0 + ansible.builtin.shell: | + {{ shell_header }} + DCN_SSH="{{ item.ssh }}" + DCN_SERVICES=({% for svc in dcn_storage_services %}"{{ svc }}" {% endfor %}) + + echo "Stopping services on DCN node: {{ item.ssh }}" + for service in ${DCN_SERVICES[*]}; do + echo "Stopping $service" + if $DCN_SSH sudo systemctl is-active $service; then + $DCN_SSH sudo systemctl stop $service + fi + done + + echo "Checking services on DCN node: {{ item.ssh }}" + for service in ${DCN_SERVICES[*]}; do + if ! $DCN_SSH systemctl show $service | grep ActiveState=inactive >/dev/null; then + echo "ERROR: Service $service still running" + else + echo "OK: Service $service is not running" + fi + done + loop: "{{ dcn_storage_nodes }}" + loop_control: + label: "{{ item.ssh }}" diff --git a/tests/vars.dcn_storage.yaml b/tests/vars.dcn_storage.yaml new file mode 100644 index 000000000..501fea7ff --- /dev/null +++ b/tests/vars.dcn_storage.yaml @@ -0,0 +1,304 @@ +# DCN Storage Adoption Variables +# Based on vars.dcn_nostorage.yaml with storage backends enabled +# +# Required --skip-tags for DCN storage adoption: +# --skip-tags barbican_adoption,swift_adoption,manila_adoption,octavia_adoption +# +# Swift is not deployed in DCN storage scenario (uses Ceph RBD for Glance) + +prelaunch_test_instance: false +ping_test: false +os_cloud_name: central +controller_ssh_identity_file: "/home/zuul/.ssh/cifmw_reproducer_key" +controller1_ssh: "ssh osp-controller-0" +controller2_ssh: ":" +controller3_ssh: ":" + +# DCN compute nodes run Glance, Cinder, and etcd services that need to be stopped +# These run on ALL DCN compute nodes (not just one per site) +dcn_storage_nodes: + - ssh: "ssh osp-dcn1-compute-az1-0" + - ssh: "ssh osp-dcn1-compute-az1-1" + - ssh: "ssh osp-dcn1-compute-az1-2" + - ssh: "ssh osp-dcn2-compute-az2-0" + - ssh: "ssh osp-dcn2-compute-az2-1" + - ssh: "ssh osp-dcn2-compute-az2-2" + +# Services to stop on DCN nodes (different from central controller services) +dcn_storage_services: + - tripleo_glance_api_internal.service + - tripleo_cinder_volume.service + - tripleo_etcd.service +edpm_tuned_profile: "throughput-performance" +dataplane_os_net_config_set_route: true +edpm_ovn_bridge_mappings: ["leaf0:br-ex"] +edpm_ovn_bridge_mappings_dcn1: ["leaf1:br-ex"] +edpm_ovn_bridge_mappings_dcn2: ["leaf2:br-ex"] +edpm_sshd_allowed_ranges: + - '192.168.122.0/24' + - '192.168.133.0/24' + - '192.168.144.0/24' + +# Storage backends for DCN with Ceph +glance_backend: ceph +cinder_volume_backend: ceph +cinder_backup_backend: ceph +manila_cephfs_conf_path: /etc/ceph/central.conf +manila_cephfs_cluster_name: central +cinder_backup_ceph_patch: | + spec: + cinder: + enabled: true + template: + cinderBackups: + central: + networkAttachments: + - storage + replicas: 1 + customServiceConfig: | + [DEFAULT] + backup_driver=cinder.backup.drivers.ceph.CephBackupDriver + backup_ceph_conf=/etc/ceph/central.conf + backup_ceph_user=openstack + backup_ceph_pool=backups + storage_availability_zone=az-central +nova_libvirt_backend: ceph + +# Key name in ceph-conf-files secret (DCN uses central.conf instead of ceph.conf) +ceph_conf_secret_key: "central.conf" + +# Multi-cluster Ceph configuration for DCN +# All ceph configs are on controller at /var/lib/tripleo-config/ceph/ +# Files: central.conf, central.client.openstack.keyring, dcn1.conf, etc. +ceph_clusters: + - name: central + - name: dcn1 + - name: dcn2 + +# Glance DCN multi-backend configuration +# Central has all backends; edge sites have central + local only +glance_dcn_clusters: + - name: central + ceph_conf: /etc/ceph/central.conf + pool: images + default: true + type: split + backends: all + metallb_ip: "172.17.0.80" + - name: dcn1 + ceph_conf: /etc/ceph/dcn1.conf + pool: images + type: edge + backends: [central, dcn1] + metallb_ip: "172.17.0.81" + - name: dcn2 + ceph_conf: /etc/ceph/dcn2.conf + pool: images + type: edge + backends: [central, dcn2] + metallb_ip: "172.17.0.82" + +# Cinder DCN multi-backend configuration +# Each cluster creates a separate cinderVolume with its own availability zone +cinder_dcn_clusters: + - name: central + ceph_conf: /etc/ceph/central.conf + rbd_pool: volumes + availability_zone: az-central + - name: dcn1 + ceph_conf: /etc/ceph/dcn1.conf + rbd_pool: volumes + availability_zone: az-dcn1 + - name: dcn2 + ceph_conf: /etc/ceph/dcn2.conf + rbd_pool: volumes + availability_zone: az-dcn2 +cinder_dcn_default_az: az-central + +# Nova DCN per-site Ceph configuration +# Each site's compute nodes get a ConfigMap with local ceph conf, fsid, and glance endpoint +# The ConfigMap is referenced by a per-site OpenStackDataPlaneService (nova-custom-ceph-$SITE) +nova_dcn_clusters: + - name: central + ceph_conf: /etc/ceph/central.conf + rbd_pool: vms + glance_store_name: central + - name: dcn1 + ceph_conf: /etc/ceph/dcn1.conf + rbd_pool: vms + glance_store_name: dcn1 + - name: dcn2 + ceph_conf: /etc/ceph/dcn2.conf + rbd_pool: vms + glance_store_name: dcn2 + +neutron_config_patch: | + spec: + neutron: + enabled: true + apiOverride: + route: {} + template: + customServiceConfig: | + [DEFAULT] + dhcp_agent_notification = True + [ml2_type_vlan] + network_vlan_ranges = leaf0:1:1000,leaf1:1:1000,leaf2:1:1000 + [neutron] + physnets = leaf0,leaf1,leaf2 + override: + service: + internal: + metadata: + annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: {{ internalapi_prefix | default('172.17.0') }}.80 + spec: + type: LoadBalancer + databaseInstance: openstack + databaseAccount: neutron + secret: osp-secret + networkAttachments: + - internalapi + +netconfig_networks: + - dnsDomain: ctlplane.example.com + mtu: 1500 + name: ctlplane + serviceNetwork: ctlplane + subnets: + - allocationRanges: + - end: 192.168.122.120 + start: 192.168.122.100 + - end: 192.168.122.200 + start: 192.168.122.150 + cidr: 192.168.122.0/24 + gateway: 192.168.122.1 + name: subnet1 + - allocationRanges: + - end: 192.168.133.120 + start: 192.168.133.100 + - end: 192.168.133.200 + start: 192.168.133.150 + cidr: 192.168.133.0/24 + gateway: 192.168.133.1 + name: ctlplanedcn1 + - allocationRanges: + - end: 192.168.144.120 + start: 192.168.144.100 + - end: 192.168.144.200 + start: 192.168.144.150 + cidr: 192.168.144.0/24 + gateway: 192.168.144.1 + name: ctlplanedcn2 + - dnsDomain: internalapi.example.com + mtu: 1500 + name: internalapi + serviceNetwork: internalapi + subnets: + - allocationRanges: + - end: 172.17.0.250 + start: 172.17.0.100 + cidr: 172.17.0.0/24 + name: subnet1 + vlan: 20 + - allocationRanges: + - end: 172.17.10.250 + start: 172.17.10.100 + cidr: 172.17.10.0/24 + name: internalapidcn1 + vlan: 30 + routes: "{{ edpm_dcn1_routes.get('internalapidcn1', []) }}" + - allocationRanges: + - end: 172.17.20.250 + start: 172.17.20.100 + cidr: 172.17.20.0/24 + name: internalapidcn2 + vlan: 40 + routes: "{{ edpm_dcn2_routes.get('internalapidcn2', []) }}" + - dnsDomain: external.example.com + mtu: 1500 + name: External + serviceNetwork: external + subnets: + - allocationRanges: + - end: 10.0.0.250 + start: 10.0.0.100 + cidr: 10.0.0.0/24 + gateway: 10.0.0.1 + name: subnet1 + vlan: 44 + - dnsDomain: storage.example.com + mtu: 1500 + name: storage + serviceNetwork: storage + subnets: + - allocationRanges: + - end: 172.18.0.250 + start: 172.18.0.100 + cidr: 172.18.0.0/24 + name: subnet1 + vlan: 21 + - allocationRanges: + - end: 172.18.10.250 + start: 172.18.10.100 + cidr: 172.18.10.0/24 + name: storagedcn1 + vlan: 31 + routes: "{{ edpm_dcn1_routes.get('storagedcn1', []) }}" + - allocationRanges: + - end: 172.18.20.250 + start: 172.18.20.100 + cidr: 172.18.20.0/24 + name: storagedcn2 + vlan: 41 + routes: "{{ edpm_dcn2_routes.get('storagedcn2', []) }}" + - dnsDomain: storagemgmt.example.com + mtu: 1500 + name: storagemgmt + serviceNetwork: storagemgmt + subnets: + - allocationRanges: + - end: 172.20.0.250 + start: 172.20.0.100 + cidr: 172.20.0.0/24 + name: subnet1 + vlan: 23 + - allocationRanges: + - end: 172.20.10.250 + start: 172.20.10.100 + cidr: 172.20.10.0/24 + name: storagemgmtdcn1 + vlan: 33 + - allocationRanges: + - end: 172.20.20.250 + start: 172.20.20.100 + cidr: 172.20.20.0/24 + name: storagemgmtdcn2 + vlan: 43 + - dnsDomain: tenant.example.com + mtu: 1500 + name: tenant + serviceNetwork: tenant + subnets: + - allocationRanges: + - end: 172.19.0.250 + start: 172.19.0.100 + cidr: 172.19.0.0/24 + name: subnet1 + vlan: 22 + - allocationRanges: + - end: 172.19.10.250 + start: 172.19.10.100 + cidr: 172.19.10.0/24 + name: tenantdcn1 + vlan: 32 + routes: "{{ edpm_dcn1_routes.get('tenantdcn1', []) }}" + - allocationRanges: + - end: 172.19.20.250 + start: 172.19.20.100 + cidr: 172.19.20.0/24 + name: tenantdcn2 + vlan: 42 + routes: "{{ edpm_dcn2_routes.get('tenantdcn2', []) }}"