Consolidate ResultsBackend and Broker to clusterConfig (#786)

* fix: remove obsolete errors

* chore: move results backend and broker to clusterConfig

* chore: regenerate charts

* fix: regenerate nix build

* chore: move results backend and broker to cluster config in tests

* docs: adapt docs and examples

* test: adapt unit tests

* fix: pre-commit

* chore: adapted changelog

* chore: add warning if broker and results backend configured without a celery executor.

Author: maltesander

CHANGELOG.md

@@ -19,6 +19,9 @@
   This means you need to replace your simple database connection string with a typed struct.
   This struct is consistent between different CRDs, so that you can easily copy/paste it between stacklets.
   Read on the [Airflow database documentation](https://docs.stackable.tech/home/nightly/airflow/usage-guide/database-connections) for details ([#754]).
+- BREAKING: Renamed and moved the `celeryExecutor` broker and results backend to `clusterConfig` ([#786]).
+  The results backend `spec.celeryExecutors.resultBackend` is now `spec.clusterConfig.celeryResultsBackend`.
+  The broker `spec.celeryExecutors.broker` is now `spec.clusterConfig.celeryBroker`.
 
 ### Fixed
 
@@ -33,6 +36,7 @@
 [#777]: https://github.com/stackabletech/airflow-operator/pull/777
 [#754]: https://github.com/stackabletech/airflow-operator/pull/754
 [#784]: https://github.com/stackabletech/airflow-operator/pull/784
+[#786]: https://github.com/stackabletech/airflow-operator/pull/786
 
 ## [26.3.0] - 2026-03-16
 

Cargo.nix

@@ -15,6 +15,15 @@ spec:
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials
+    celeryResultsBackend:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecretName: airflow-postgresql-credentials
+    celeryBroker:
+      redis:
+        host: airflow-redis-master
+        credentialsSecretName: airflow-redis-credentials
   webservers:
     roleConfig:
       listenerClass: external-unstable
@@ -29,15 +38,6 @@ spec:
         envOverrides: *envOverrides
         replicas: 1
   celeryExecutors:
-    resultBackend:
-      postgresql:
-        host: airflow-postgresql
-        database: airflow
-        credentialsSecretName: airflow-postgresql-credentials
-    broker:
-      redis:
-        host: airflow-redis-master
-        credentialsSecretName: airflow-redis-credentials
     roleGroups:
       default:
         envOverrides: *envOverrides

crate-hashes.json

@@ -16,22 +16,22 @@ spec:
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials
-  webservers:
-    roleConfig:
-      listenerClass: external-unstable
-    roleGroups:
-      default:
-        replicas: 1
-  celeryExecutors:
-    resultBackend:
+    celeryResultsBackend:
       postgresql:
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials
-    broker:
+    celeryBroker:
       redis:
         host: airflow-redis-master
         credentialsSecretName: airflow-redis-credentials
+  webservers:
+    roleConfig:
+      listenerClass: external-unstable
+    roleGroups:
+      default:
+        replicas: 1
+  celeryExecutors:
     roleGroups:
       default:
         replicas: 1

docs/modules/airflow/examples/example-airflow-incluster.yaml

@@ -20,22 +20,26 @@ spec:
 <2> A reference to a Secret which must contain the two fields `username` and `password`.
 
 The queue/broker metadata and URL is only needed when running the celery executor.
-The `resultBackend` definition uses the same structure as `metadataDatabase` shown above.
-The `broker` definition requires Redis connection details.
+The `celeryResultsBackend` definition uses the same structure as `metadataDatabase` shown above.
+The `celeryBroker` definition requires Redis connection details.
 
 [source,yaml]
 ----
 spec:
-  celeryExecutors:
-    resultBackend:
+  clusterConfig:
+    celeryResultsBackend:
       postgresql: # <1>
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials # <2>
-    broker:
+    celeryBroker:
       redis: # <3>
         host: airflow-redis-master
         credentialsSecretName: airflow-redis-credentials # <2>
+  celeryExecutors:
+    roleGroups:
+      default:
+        replicas: 1
 ----
 <1> A reference to one of the supported database backends (e.g. `postgresql`).
 <2> A reference to a secret which must contain the two fields `username` and `password`.
@@ -57,10 +61,11 @@ spec:
 [source,yaml]
 ----
 spec:
-  resultBackend:
+  clusterConfig:
+    celeryResultsBackend:
       generic:
         connectionUrlSecretName: postgresql-celery # <2>
-    broker:
+    celeryBroker:
       generic:
         connectionUrlSecretName: redis-celery # <3>
 ----