diff --git a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml index eb0d9e0b57..b8e6a34d77 100644 --- a/apis/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/apis/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -9512,6 +9512,9 @@ spec: additionalProperties: type: string type: object + notificationsBus: + default: rabbitmq + type: string nova: properties: apiOverride: diff --git a/apis/core/v1beta1/openstackcontrolplane_types.go b/apis/core/v1beta1/openstackcontrolplane_types.go index b0cf1cb52e..1524c8968a 100644 --- a/apis/core/v1beta1/openstackcontrolplane_types.go +++ b/apis/core/v1beta1/openstackcontrolplane_types.go @@ -126,6 +126,14 @@ type OpenStackControlPlaneSpec struct { // Rabbitmq - Parameters related to the Rabbitmq service Rabbitmq RabbitmqSection `json:"rabbitmq,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=rabbitmq + // NotificationsBus - the name of RabbitMQ Cluster CR to select a Messaging + // Bus Service instance used by all services that produce or consume notifications. + // Avoid colocating it with RabbitMQ services used for PRC. + // That instance will be pushed down for services, unless overriden in templates. + NotificationsBus string `json:"notificationsBus"` + // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec // Memcached - Parameters related to the Memcached service @@ -484,7 +492,7 @@ type RabbitmqSection struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec - // Templates - Overrides to use when creating the Rabbitmq clusters + // Templates - Overrides to use when creating the Rabbitmq clusters for RPC and (optionally) Notifications. Templates *map[string]rabbitmqv1.RabbitMqSpecCore `json:"templates"` } diff --git a/bindata/crds/crds.yaml b/bindata/crds/crds.yaml index b48d58b293..71d76fe12f 100644 --- a/bindata/crds/crds.yaml +++ b/bindata/crds/crds.yaml @@ -9676,6 +9676,9 @@ spec: additionalProperties: type: string type: object + notificationsBus: + default: rabbitmq + type: string nova: properties: apiOverride: diff --git a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml index eb0d9e0b57..b8e6a34d77 100644 --- a/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml +++ b/config/crd/bases/core.openstack.org_openstackcontrolplanes.yaml @@ -9512,6 +9512,9 @@ spec: additionalProperties: type: string type: object + notificationsBus: + default: rabbitmq + type: string nova: properties: apiOverride: diff --git a/go.mod b/go.mod index 95d390e8a3..f573f5e627 100644 --- a/go.mod +++ b/go.mod @@ -126,4 +126,4 @@ replace github.com/openstack-k8s-operators/openstack-operator/apis => ./apis replace github.com/openshift/api => github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 //allow-merging // custom RabbitmqClusterSpecCore for OpenStackControlplane (v2.9.0_patches_tag) -replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20241017142550-a3524acedd49 //allow-merging +replace github.com/rabbitmq/cluster-operator/v2 => github.com/openstack-k8s-operators/rabbitmq-cluster-operator/v2 v2.6.1-0.20241017142550-a3524acedd49 //allow-merging \ No newline at end of file diff --git a/tests/functional/ctlplane/base_test.go b/tests/functional/ctlplane/base_test.go index 23f82d09d0..3ee8264e54 100644 --- a/tests/functional/ctlplane/base_test.go +++ b/tests/functional/ctlplane/base_test.go @@ -57,6 +57,8 @@ type Names struct { NeutronName types.NamespacedName HorizonName types.NamespacedName HeatName types.NamespacedName + NovaName types.NamespacedName + PlacementName types.NamespacedName TelemetryName types.NamespacedName DBName types.NamespacedName DBCertName types.NamespacedName @@ -64,31 +66,33 @@ type Names struct { DBCell1CertName types.NamespacedName RabbitMQName types.NamespacedName RabbitMQCertName types.NamespacedName + RabbitMQNotificationsCertName types.NamespacedName RabbitMQCell1Name types.NamespacedName RabbitMQCell1CertName types.NamespacedName NoVNCProxyCell1CertPublicRouteName types.NamespacedName NoVNCProxyCell1CertPublicSvcName types.NamespacedName NoVNCProxyCell1CertVencryptName types.NamespacedName - ServiceAccountName types.NamespacedName - RoleName types.NamespacedName - RoleBindingName types.NamespacedName - RootCAPublicName types.NamespacedName - RootCAInternalName types.NamespacedName - RootCAOvnName types.NamespacedName - RootCALibvirtName types.NamespacedName - SelfSignedIssuerName types.NamespacedName - CustomIssuerName types.NamespacedName - CustomServiceCertSecretName types.NamespacedName - CABundleName types.NamespacedName - OpenStackClientName types.NamespacedName - OVNNorthdName types.NamespacedName - OVNNorthdCertName types.NamespacedName - OVNControllerName types.NamespacedName - OVNControllerCertName types.NamespacedName - OVNDbServerNBName types.NamespacedName - OVNDbServerSBName types.NamespacedName - NeutronOVNCertName types.NamespacedName - OpenStackTopology []types.NamespacedName + RabbitMQNotificationsName types.NamespacedName + ServiceAccountName types.NamespacedName + RoleName types.NamespacedName + RoleBindingName types.NamespacedName + RootCAPublicName types.NamespacedName + RootCAInternalName types.NamespacedName + RootCAOvnName types.NamespacedName + RootCALibvirtName types.NamespacedName + SelfSignedIssuerName types.NamespacedName + CustomIssuerName types.NamespacedName + CustomServiceCertSecretName types.NamespacedName + CABundleName types.NamespacedName + OpenStackClientName types.NamespacedName + OVNNorthdName types.NamespacedName + OVNNorthdCertName types.NamespacedName + OVNControllerName types.NamespacedName + OVNControllerCertName types.NamespacedName + OVNDbServerNBName types.NamespacedName + OVNDbServerSBName types.NamespacedName + NeutronOVNCertName types.NamespacedName + OpenStackTopology []types.NamespacedName } func CreateNames(openstackControlplaneName types.NamespacedName) Names { @@ -176,6 +180,14 @@ func CreateNames(openstackControlplaneName types.NamespacedName) Names { Namespace: openstackControlplaneName.Namespace, Name: "telemetry", }, + NovaName: types.NamespacedName{ + Namespace: openstackControlplaneName.Namespace, + Name: "nova", + }, + PlacementName: types.NamespacedName{ + Namespace: openstackControlplaneName.Namespace, + Name: "placement", + }, DBName: types.NamespacedName{ Namespace: openstackControlplaneName.Namespace, Name: "openstack", @@ -200,10 +212,18 @@ func CreateNames(openstackControlplaneName types.NamespacedName) Names { Namespace: openstackControlplaneName.Namespace, Name: "cert-rabbitmq-svc", }, + RabbitMQNotificationsCertName: types.NamespacedName{ + Namespace: openstackControlplaneName.Namespace, + Name: "cert-rabbitmq-notifications-svc", + }, RabbitMQCell1Name: types.NamespacedName{ Namespace: openstackControlplaneName.Namespace, Name: "rabbitmq-cell1", }, + RabbitMQNotificationsName: types.NamespacedName{ + Namespace: openstackControlplaneName.Namespace, + Name: "rabbitmq-notifications", + }, RabbitMQCell1CertName: types.NamespacedName{ Namespace: openstackControlplaneName.Namespace, Name: "cert-rabbitmq-cell1-svc", @@ -504,6 +524,9 @@ func GetDefaultOpenStackControlPlaneSpec() map[string]interface{} { names.RabbitMQCell1Name.Name: map[string]interface{}{ "replicas": 1, }, + names.RabbitMQNotificationsName.Name: map[string]interface{}{ + "replicas": 1, + }, } galeraTemplate := map[string]interface{}{ names.DBName.Name: map[string]interface{}{ @@ -614,7 +637,9 @@ func GetDefaultOpenStackControlPlaneSpec() map[string]interface{} { "barbican": map[string]interface{}{ "enabled": false, }, - "openstackclient": map[string]interface{}{}, + // "openstackclient": map[string]interface{}{ + // "enabled": true, + // }, "manila": map[string]interface{}{ "enabled": true, "template": manilaTemplate, diff --git a/tests/functional/ctlplane/openstackoperator_controller_test.go b/tests/functional/ctlplane/openstackoperator_controller_test.go index c81106611a..296fb575f4 100644 --- a/tests/functional/ctlplane/openstackoperator_controller_test.go +++ b/tests/functional/ctlplane/openstackoperator_controller_test.go @@ -30,6 +30,7 @@ import ( k8s_corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -40,6 +41,7 @@ import ( cinderv1 "github.com/openstack-k8s-operators/cinder-operator/api/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/certmanager" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -830,6 +832,7 @@ var _ = Describe("OpenStackOperator controller", func() { BeforeEach(func() { // create cert secrets for rabbitmq instances DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQNotificationsCertName)) DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) // create cert secrets for memcached instance DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.MemcachedCertName)) @@ -1143,6 +1146,7 @@ var _ = Describe("OpenStackOperator controller", func() { BeforeEach(func() { // create cert secrets for rabbitmq instances DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQNotificationsCertName)) DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) // create cert secrets for memcached instance DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.MemcachedCertName)) @@ -1270,6 +1274,7 @@ var _ = Describe("OpenStackOperator controller", func() { BeforeEach(func() { // create cert secrets for rabbitmq instances DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQNotificationsCertName)) DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) // create cert secrets for memcached instance DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.MemcachedCertName)) @@ -1793,6 +1798,160 @@ var _ = Describe("OpenStackOperator controller", func() { }) }) + When("A OpenStackControlplane instance is created with top-scope notifications config pushed down", func() { + BeforeEach(func() { + // NOTE(bogdando): DBs certs need to be created here as well, but those are already existing somehow + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQNotificationsCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.MemcachedCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.OVNNorthdCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.OVNControllerCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.NeutronOVNCertName)) + + DeferCleanup(k8sClient.Delete, ctx, + th.CreateSecret(types.NamespacedName{Name: "openstack-config-secret", Namespace: namespace}, map[string][]byte{"secure.yaml": []byte("foo")})) + DeferCleanup(k8sClient.Delete, ctx, + th.CreateConfigMap(types.NamespacedName{Name: "openstack-config", Namespace: namespace}, map[string]interface{}{"clouds.yaml": string("foo"), "OS_CLOUD": "default"})) + + spec := GetDefaultOpenStackControlPlaneSpec() + spec["notificationsBus"] = "rabbitmq-notifications" + // enable dependencies + spec["nova"] = map[string]interface{}{ + "enabled": true, + "template": map[string]interface{}{ + "apiTimeout": 60, + "cellTemplates": map[string]interface{}{ + "cell0": map[string]interface{}{}, + }, + }, + } + spec["galera"] = map[string]interface{}{ + "enabled": true, + } + spec["memcached"] = map[string]interface{}{ + "enabled": true, + "templates": map[string]interface{}{ + "memcached": map[string]interface{}{ + "replicas": 1, + }, + }, + } + spec["rabbitmq"] = map[string]interface{}{ + "enabled": true, + "templates": map[string]interface{}{ + "rabbitmq": map[string]interface{}{ + "replicas": 1, + }, + }, + } + spec["keystone"] = map[string]interface{}{ + "enabled": true, + } + spec["glance"] = map[string]interface{}{ + "enabled": true, + } + spec["neutron"] = map[string]interface{}{ + "enabled": true, + } + spec["placement"] = map[string]interface{}{ + "enabled": true, + "template": map[string]interface{}{ + "apiTimeout": 60, + }, + } + // turn off unrelated to this test case services + spec["horizon"] = map[string]interface{}{ + "enabled": false, + } + spec["cinder"] = map[string]interface{}{ + "enabled": false, + } + spec["swift"] = map[string]interface{}{ + "enabled": false, + } + spec["redis"] = map[string]interface{}{ + "enabled": false, + } + spec["ironic"] = map[string]interface{}{ + "enabled": false, + } + spec["designate"] = map[string]interface{}{ + "enabled": false, + } + spec["barbican"] = map[string]interface{}{ + "enabled": false, + } + spec["manila"] = map[string]interface{}{ + "enabled": false, + } + spec["heat"] = map[string]interface{}{ + "enabled": false, + } + spec["telemetry"] = map[string]interface{}{ + "enabled": false, + } + + Eventually(func(g Gomega) { + g.Expect(CreateOpenStackControlPlane(names.OpenStackControlplaneName, spec)).Should(Not(BeNil())) + keystoneAPI := keystone.GetKeystoneAPI(names.KeystoneAPIName) + g.Expect(keystoneAPI).Should(Not(BeNil())) + SimulateControlplaneReady() + }, timeout, interval).Should(Succeed()) + + DeferCleanup( + th.DeleteInstance, + GetOpenStackControlPlane(names.OpenStackControlplaneName), + ) + + Eventually(func(g Gomega) { + OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) + OSCtlplane.Status.ObservedGeneration = OSCtlplane.Generation + OSCtlplane.Status.Conditions.MarkTrue(corev1.OpenStackControlPlaneMemcachedReadyCondition, "Ready") + OSCtlplane.Status.Conditions.MarkTrue(corev1.OpenStackControlPlaneRabbitMQReadyCondition, "Ready") + OSCtlplane.Status.Conditions.MarkTrue(corev1.OpenStackControlPlaneNeutronReadyCondition, "Ready") + OSCtlplane.Status.Conditions.MarkTrue(corev1.OpenStackControlPlaneGlanceReadyCondition, "Ready") + OSCtlplane.Status.Conditions.MarkTrue(corev1.OpenStackControlPlanePlacementAPIReadyCondition, "Ready") + g.Expect(k8sClient.Update(ctx, OSCtlplane)).Should(Succeed()) + th.Logger.Info("Simulated nova dependencies ready", "on", names.OpenStackControlplaneName) + }, timeout, interval).Should(Succeed()) + + // nova to become ready + Eventually(func(g Gomega) { + conditions := OpenStackControlPlaneConditionGetter(names.OpenStackControlplaneName) + g.Expect(conditions.Has(corev1.OpenStackControlPlaneNovaReadyCondition)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("should have configured nova notifications bus instance from top-scope", func() { + nova := &novav1.Nova{} + notificationBusName := "rabbitmq-notifications" + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, names.NovaName, nova)).Should(Succeed()) + g.Expect(nova).ShouldNot(BeNil()) + g.Expect(nova.Spec.NotificationsBusInstance).Should(Equal(¬ificationBusName)) + }, timeout, interval).Should(Succeed()) + }) + + It("should have configured nova notifications bus instance from the service template", func() { + OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) + overrideBusName := "rabbitmq-custom" + Eventually(func(g Gomega) { + // force-update the notifications bus instance, unless it is overriden in templates + OSCtlplane.Spec.NotificationsBus = "" + OSCtlplane.Spec.Nova.Template.NotificationsBusInstance = &overrideBusName + g.Expect(k8sClient.Update(ctx, OSCtlplane)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + + nova := &novav1.Nova{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, names.NovaName, nova)).Should(Succeed()) + g.Expect(nova).ShouldNot(BeNil()) + g.Expect(nova.Spec.NotificationsBusInstance).Should(Equal(&overrideBusName)) + }, timeout, interval).Should(Succeed()) + }) + }) + When("OpenStackControlplane instance is deleted", func() { BeforeEach(func() { DeferCleanup( @@ -2147,6 +2306,7 @@ var _ = Describe("OpenStackOperator controller", func() { // create cert secrets for rabbitmq instances DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQNotificationsCertName)) DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) // create cert secrets for memcached instance DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.MemcachedCertName)) @@ -2987,6 +3147,7 @@ var _ = Describe("OpenStackOperator controller nova cell deletion", func() { // create cert secrets for rabbitmq instances th.CreateCertSecret(names.RabbitMQCertName) + th.CreateCertSecret(names.RabbitMQNotificationsCertName) th.CreateCertSecret(names.RabbitMQCell1CertName) // create cert secrets for ovn instance @@ -3130,17 +3291,17 @@ var _ = Describe("OpenStackOperator controller nova cell deletion", func() { Eventually(func(g Gomega) { OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) rabbitTemplates := *(OSCtlplane.Spec.Rabbitmq.Templates) - g.Expect(rabbitTemplates).Should(HaveLen(2)) + g.Expect(rabbitTemplates).Should(HaveLen(3)) delete(rabbitTemplates, names.RabbitMQCell1Name.Name) OSCtlplane.Spec.Rabbitmq.Templates = &rabbitTemplates g.Expect(k8sClient.Update(ctx, OSCtlplane)).Should(Succeed()) }, timeout, interval).Should(Succeed()) - // Only 1 cell in rabbitmq template + // Only 1 cell and notifications bus in rabbitmq template Eventually(func(g Gomega) { OSCtlplane := GetOpenStackControlPlane(names.OpenStackControlplaneName) rabbitTemplates := *(OSCtlplane.Spec.Rabbitmq.Templates) - g.Expect(rabbitTemplates).Should(HaveLen(1)) + g.Expect(rabbitTemplates).Should(HaveLen(2)) }, timeout, interval).Should(Succeed()) // cell1.rabbitmq should not exists in db diff --git a/tests/functional/ctlplane/openstackversion_controller_test.go b/tests/functional/ctlplane/openstackversion_controller_test.go index d6e92eeb0b..fe337686b3 100644 --- a/tests/functional/ctlplane/openstackversion_controller_test.go +++ b/tests/functional/ctlplane/openstackversion_controller_test.go @@ -275,6 +275,7 @@ var _ = Describe("OpenStackOperator controller", func() { // create cert secrets for rabbitmq instances DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCertName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQNotificationsCertName)) DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.RabbitMQCell1CertName)) // (mschuppert) create root CA secrets as there is no certmanager running.