merge main

Author: adwk67

CHANGELOG.md

@@ -19,6 +19,9 @@
   This means you need to replace your simple database connection string with a typed struct.
   This struct is consistent between different CRDs, so that you can easily copy/paste it between stacklets.
   Read on the [Airflow database documentation](https://docs.stackable.tech/home/nightly/airflow/usage-guide/database-connections) for details ([#754]).
+- BREAKING: Renamed and moved the `celeryExecutor` broker and results backend to `clusterConfig` ([#786]).
+  The results backend `spec.celeryExecutors.resultBackend` is now `spec.clusterConfig.celeryResultsBackend`.
+  The broker `spec.celeryExecutors.broker` is now `spec.clusterConfig.celeryBroker`.
 
 ### Fixed
 
@@ -33,6 +36,7 @@
 [#777]: https://github.com/stackabletech/airflow-operator/pull/777
 [#754]: https://github.com/stackabletech/airflow-operator/pull/754
 [#784]: https://github.com/stackabletech/airflow-operator/pull/784
+[#786]: https://github.com/stackabletech/airflow-operator/pull/786
 
 ## [26.3.0] - 2026-03-16
 

Tiltfile

@@ -2,13 +2,22 @@
 meta = read_json('nix/meta.json')
 operator_name = meta['operator']['name']
 
-# If tilt_options.json exists read it and load the default_registry and default_repository value from it
+# If tilt_options.json exists read it and load the default_registry, default_operator_repository,
+# and default_product_repository value from it
 settings = read_json('tilt_options.json', default={})
 registry = settings.get('default_registry', 'oci.stackable.tech')
-repository = settings.get('default_repository', registry + '/' + 'sdp')
-operator_image_name = repository + '/' + operator_name
 
-# Configure default registry either read from config file above, or with default value of "oci.stackable.tech"
+# Construct the operator image repository. It uses the "sandbox" instead of the "sdp" namespace to
+# separate "testing/local" images from official (published) images.
+operator_repository = settings.get('default_operator_repository', registry + '/' + 'sandbox')
+operator_image_name = operator_repository + '/' + operator_name
+
+# For the product image, we wanna use the images in the "sdp" namespace, because "sandbox" doesn't
+# contain those images (by default).
+product_repository = settings.get('default_product_repository', registry + '/' + 'sdp')
+
+# Configure default registry either read from config file above, or with default value of
+# "oci.stackable.tech"
 default_registry(registry)
 
 custom_build(
@@ -31,14 +40,16 @@ k8s_kind('DaemonSet', image_json_path='{.spec.template.metadata.annotations.inte
 # supported by helm(set).
 helm_values = settings.get('helm_values', None)
 
-helm_override_image_repository = 'image.repository=' + repository
+helm_override_operator_image_repository = 'image.repository=' + operator_repository
+helm_override_product_image_repository = 'image.productRepository=' + product_repository
 
 k8s_yaml(helm(
    'deploy/helm/' + operator_name,
    name=operator_name,
    namespace="stackable-operators",
    set=[
-      helm_override_image_repository,
+      helm_override_operator_image_repository,
+      helm_override_product_image_repository,
    ],
    values=helm_values,
 ))

deploy/helm/airflow-operator/templates/_helpers.tpl

@@ -79,7 +79,7 @@ helm.sh/test: {{ include "operator.chart" . }}
 {{- end }}
 
 {{/*
-Build the full container image reference.
+Build the full operator container image reference.
 */}}
 {{- define "operator.image" -}}
 {{- printf "%s/%s:%s" .Values.image.repository .Chart.Name (.Values.image.tag | default .Chart.AppVersion) -}}

deploy/helm/airflow-operator/templates/deployment.yaml

@@ -70,9 +70,9 @@ spec:
             - name: OPERATOR_SERVICE_NAME
               value: {{ include "operator.fullname" . }}
 
-            # The image repository, like "oci.stackable.tech/sdp"
+            # The product image repository, like "oci.stackable.tech/sdp".
             - name: IMAGE_REPOSITORY
-              value: {{ .Values.image.repository }}
+              value: {{ .Values.image.productRepository | default .Values.image.repository }}
 
             # Operators need to know the node name they are running on, to e.g. discover the
             # Kubernetes domain name from the kubelet API.

docker/Dockerfile

@@ -37,7 +37,7 @@ ARG STACKABLE_USER_NAME="stackable"
 # "-c": Allows the execution of commands passed as a string
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
-# These labels have mostly been superceded by the OpenContainer spec annotations below but it doesn't hurt to include them
+# These labels have mostly been superseded by the OpenContainer spec annotations below but it doesn't hurt to include them
 # http://label-schema.org/rc1/
 LABEL name="Stackable Operator for Apache Airflow"
 LABEL maintainer="info@stackable.tech"

docs/modules/airflow/examples/example-airflow-incluster.yaml

@@ -15,6 +15,15 @@ spec:
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials
+    celeryResultsBackend:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecretName: airflow-postgresql-credentials
+    celeryBroker:
+      redis:
+        host: airflow-redis-master
+        credentialsSecretName: airflow-redis-credentials
   webservers:
     roleConfig:
       listenerClass: external-unstable
@@ -29,15 +38,6 @@ spec:
         envOverrides: *envOverrides
         replicas: 1
   celeryExecutors:
-    resultBackend:
-      postgresql:
-        host: airflow-postgresql
-        database: airflow
-        credentialsSecretName: airflow-postgresql-credentials
-    broker:
-      redis:
-        host: airflow-redis-master
-        credentialsSecretName: airflow-redis-credentials
     roleGroups:
       default:
         envOverrides: *envOverrides

docs/modules/airflow/examples/getting_started/code/airflow.yaml

@@ -16,22 +16,22 @@ spec:
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials
-  webservers:
-    roleConfig:
-      listenerClass: external-unstable
-    roleGroups:
-      default:
-        replicas: 1
-  celeryExecutors:
-    resultBackend:
+    celeryResultsBackend:
       postgresql:
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials
-    broker:
+    celeryBroker:
       redis:
         host: airflow-redis-master
         credentialsSecretName: airflow-redis-credentials
+  webservers:
+    roleConfig:
+      listenerClass: external-unstable
+    roleGroups:
+      default:
+        replicas: 1
+  celeryExecutors:
     roleGroups:
       default:
         replicas: 1

docs/modules/airflow/pages/usage-guide/database-connections.adoc

@@ -20,22 +20,26 @@ spec:
 <2> A reference to a Secret which must contain the two fields `username` and `password`.
 
 The queue/broker metadata and URL is only needed when running the celery executor.
-The `resultBackend` definition uses the same structure as `metadataDatabase` shown above.
-The `broker` definition requires Redis connection details.
+The `celeryResultsBackend` definition uses the same structure as `metadataDatabase` shown above.
+The `celeryBroker` definition requires Redis connection details.
 
 [source,yaml]
 ----
 spec:
-  celeryExecutors:
-    resultBackend:
+  clusterConfig:
+    celeryResultsBackend:
       postgresql: # <1>
         host: airflow-postgresql
         database: airflow
         credentialsSecretName: airflow-postgresql-credentials # <2>
-    broker:
+    celeryBroker:
       redis: # <3>
         host: airflow-redis-master
         credentialsSecretName: airflow-redis-credentials # <2>
+  celeryExecutors:
+    roleGroups:
+      default:
+        replicas: 1
 ----
 <1> A reference to one of the supported database backends (e.g. `postgresql`).
 <2> A reference to a secret which must contain the two fields `username` and `password`.
@@ -57,10 +61,11 @@ spec:
 [source,yaml]
 ----
 spec:
-  resultBackend:
+  clusterConfig:
+    celeryResultsBackend:
       generic:
         connectionUrlSecretName: postgresql-celery # <2>
-    broker:
+    celeryBroker:
       generic:
         connectionUrlSecretName: redis-celery # <3>
 ----