Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
- The `runAsUser` and `runAsGroup` fields will not be set anymore by the operator
- The defaults from the docker images itself will now apply, which will be different from 1000/0 going forward
- This is marked as breaking because tools and policies might exist, which require these fields to be set
- Changed listener class to be role-only ([#645]).

### Fixed

Expand All @@ -48,6 +49,7 @@
[#625]: https://github.com/stackabletech/airflow-operator/pull/625
[#630]: https://github.com/stackabletech/airflow-operator/pull/630
[#636]: https://github.com/stackabletech/airflow-operator/pull/636
[#645]: https://github.com/stackabletech/airflow-operator/pull/645

## [25.3.0] - 2025-03-21

Expand Down
13 changes: 5 additions & 8 deletions deploy/helm/airflow-operator/crds/crds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1332,10 +1332,6 @@ spec:
description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
nullable: true
type: string
listenerClass:
description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver.
nullable: true
type: string
logging:
default:
containers: {}
Expand Down Expand Up @@ -1480,11 +1476,16 @@ spec:
x-kubernetes-preserve-unknown-fields: true
roleConfig:
default:
listenerClass: cluster-internal
podDisruptionBudget:
enabled: true
maxUnavailable: null
description: This is a product-agnostic RoleConfig, which is sufficient for most of the products.
properties:
listenerClass:
default: cluster-internal
description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver.
type: string
podDisruptionBudget:
default:
enabled: true
Expand Down Expand Up @@ -1553,10 +1554,6 @@ spec:
description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
nullable: true
type: string
listenerClass:
description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver.
nullable: true
type: string
logging:
default:
containers: {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ spec:
mountPath: /dags/test_airflow_dag.py # <6>
subPath: test_airflow_dag.py # <7>
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ spec:
exposeConfig: false
credentialsSecret: simple-airflow-credentials
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ spec:
productVersion: 2.10.5
clusterConfig: {}
webservers:
config:
roleConfig:
listenerClass: external-unstable
envOverrides: &envOverrides
AIRFLOW__LOGGING__REMOTE_LOGGING: "True"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ spec:
productVersion: 2.10.5
clusterConfig: {}
webservers:
config:
roleConfig:
listenerClass: external-unstable
envOverrides: &envOverrides
AIRFLOW__CORE__XCOM_BACKEND: airflow.providers.common.io.xcom.backend.XComObjectStorageBackend
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ spec:
exposeConfig: false
credentialsSecret: simple-airflow-credentials
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
8 changes: 5 additions & 3 deletions docs/modules/airflow/pages/usage-guide/listenerclass.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,20 @@
:description: Configure Airflow service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable.

The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Webserver pod.
The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.config.listenerClass`:
The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.roleConfig.listenerClass`:

[source,yaml]
----
spec:
webservers:
config:
roleConfig:
listenerClass: external-unstable # <1>
config:
...
schedulers:
...
celeryExecutors:
...
----
<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`).
<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`) at role-level.
This can be set only for the webservers role.
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster-dags-cmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ spec:
mountPath: /dags/test_airflow_dag.py
subPath: test_airflow_dag.py
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster-ldap-insecure-tls.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ spec:
- authenticationClass: airflow-with-ldap-insecure-tls-ldap
userRegistrationRole: Admin
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster-ldap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ spec:
- authenticationClass: airflow-with-ldap-server-veri-tls-ldap
userRegistrationRole: Admin
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
2 changes: 1 addition & 1 deletion examples/simple-airflow-cluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ spec:
exposeConfig: false
credentialsSecret: simple-airflow-credentials
webservers:
config:
roleConfig:
listenerClass: external-unstable
roleGroups:
default:
Expand Down
58 changes: 26 additions & 32 deletions rust/operator-binary/src/airflow_controller.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ use stackable_operator::{
core::{DeserializeGuard, error_boundary},
runtime::{controller::Action, reflector::ObjectRef},
},
kvp::{Annotation, Label, LabelError, Labels},
kvp::{Annotation, Label, LabelError, Labels, ObjectLabels},
logging::controller::ReconcilerError,
product_config_utils::{
CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, env_vars_from,
Expand Down Expand Up @@ -512,26 +512,6 @@ pub async fn reconcile_airflow(
&git_sync_resources,
)?;

if let Some(listener_class) =
airflow.merged_listener_class(&airflow_role, &rolegroup.role_group)
{
if let Some(listener_group_name) =
airflow.group_listener_name(&airflow_role, &rolegroup)
{
let rg_group_listener = build_group_listener(
airflow,
&resolved_product_image,
&rolegroup,
listener_class.to_string(),
listener_group_name,
)?;
cluster_resources
.add(client, rg_group_listener)
.await
.context(ApplyGroupListenerSnafu)?;
}
}

ss_cond_builder.add(
cluster_resources
.add(client, rg_statefulset)
Expand Down Expand Up @@ -568,6 +548,27 @@ pub async fn reconcile_airflow(
.await
.context(FailedToCreatePdbSnafu)?;
}

if let Some(listener_class) = airflow_role.listener_class_name(airflow) {
if let Some(listener_group_name) = airflow.group_listener_name(&airflow_role) {
let rg_group_listener = build_group_listener(
airflow,
build_recommended_labels(
airflow,
AIRFLOW_CONTROLLER_NAME,
&resolved_product_image.app_version_label,
role_name,
"none",
),
listener_class.to_string(),
listener_group_name,
)?;
cluster_resources
.add(client, rg_group_listener)
.await
.context(ApplyGroupListenerSnafu)?;
}
}
}

cluster_resources
Expand Down Expand Up @@ -840,8 +841,7 @@ fn build_rolegroup_metadata(

pub fn build_group_listener(
airflow: &v1alpha1::AirflowCluster,
resolved_product_image: &ResolvedProductImage,
rolegroup: &RoleGroupRef<v1alpha1::AirflowCluster>,
object_labels: ObjectLabels<v1alpha1::AirflowCluster>,
listener_class: String,
listener_group_name: String,
) -> Result<listener::v1alpha1::Listener> {
Expand All @@ -851,13 +851,7 @@ pub fn build_group_listener(
.name(listener_group_name)
.ownerreference_from_resource(airflow, None, Some(true))
.context(ObjectMissingMetadataForOwnerRefSnafu)?
.with_recommended_labels(build_recommended_labels(
airflow,
AIRFLOW_CONTROLLER_NAME,
&resolved_product_image.app_version_label,
&rolegroup.role,
&rolegroup.role_group,
))
.with_recommended_labels(object_labels)
.context(ObjectMetaSnafu)?
.build(),
spec: listener::v1alpha1::ListenerSpec {
Expand Down Expand Up @@ -916,7 +910,7 @@ fn build_server_rolegroup_statefulset(
// A version value is required, and we do want to use the "recommended" format for the other desired labels
"none",
&rolegroup_ref.role,
&rolegroup_ref.role_group,
"none",
Comment thread
adwk67 marked this conversation as resolved.
Outdated
))
.context(LabelBuildSnafu)?;

Expand Down Expand Up @@ -1019,7 +1013,7 @@ fn build_server_rolegroup_statefulset(

let mut pvcs: Option<Vec<PersistentVolumeClaim>> = None;

if let Some(listener_group_name) = airflow.group_listener_name(airflow_role, rolegroup_ref) {
if let Some(listener_group_name) = airflow.group_listener_name(airflow_role) {
// Listener endpoints for the Webserver role will use persistent volumes
// so that load balancers can hard-code the target addresses. This will
// be the case even when no class is set (and the value defaults to
Expand Down
Loading