From 910db9387c4e8525a7ba00cf4d8ffaa1efda757f Mon Sep 17 00:00:00 2001 From: Miretpl Date: Sun, 5 Apr 2026 15:13:03 +0200 Subject: [PATCH 1/3] Refactor workers extraInitContainers-related tests --- .../airflow_aux/test_pod_template_file.py | 18 +++++++++------ .../helm_tests/airflow_core/test_worker.py | 23 ++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py index 70a4f751cb8ce..7b47e0af468f3 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -1080,10 +1080,12 @@ def test_should_add_extra_init_containers(self): chart_dir=self.temp_chart_dir, ) - assert jmespath.search("spec.initContainers[-1]", docs[0]) == { - "name": "test-init-container", - "image": "test-registry/test-repo:test-tag", - } + assert jmespath.search("spec.initContainers", docs[0]) == [ + { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } + ] def test_should_template_extra_init_containers(self): docs = render_chart( @@ -1096,9 +1098,11 @@ def test_should_template_extra_init_containers(self): chart_dir=self.temp_chart_dir, ) - assert jmespath.search("spec.initContainers[-1]", docs[0]) == { - "name": "release-name-test-init-container", - } + assert jmespath.search("spec.initContainers", docs[0]) == [ + { + "name": "release-name-test-init-container", + } + ] @pytest.mark.parametrize( "workers_values", diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index 09ca0ee151c6a..eb3286f896218 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -291,10 +291,13 @@ def test_should_add_extra_init_containers(self): show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) == { - "name": "test-init-container", - "image": "test-registry/test-repo:test-tag", - } + # [1:] -> Skipping wait-for-airflow-migrations init container + assert jmespath.search("spec.template.spec.initContainers[1:]", docs[0]) == [ + { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } + ] def test_should_template_extra_init_containers(self): docs = render_chart( @@ -306,9 +309,10 @@ def test_should_template_extra_init_containers(self): show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) == { - "name": "release-name-test-init-container" - } + # [1:] -> Skipping wait-for-airflow-migrations init container + assert jmespath.search("spec.template.spec.initContainers[1:]", docs[0]) == [ + {"name": "release-name-test-init-container"} + ] def test_should_add_extra_volume_and_extra_volume_mount(self): docs = render_chart( @@ -952,7 +956,10 @@ def test_extra_init_container_restart_policy_is_configurable(self): show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.initContainers[1].restartPolicy", docs[0]) == "Always" + # [1:] -> Skipping wait-for-airflow-migrations init container + assert jmespath.search("spec.template.spec.initContainers[1:] | [*].restartPolicy", docs[0]) == [ + "Always" + ] @pytest.mark.parametrize( ("log_values", "expected_volume"), From c8c39f28a63dbcc153e7ba66976bb5406eb47c74 Mon Sep 17 00:00:00 2001 From: Miretpl Date: Sun, 5 Apr 2026 15:22:36 +0200 Subject: [PATCH 2/3] Add workers.celery.extraInitContainers & workers.kubernetes.extraInitContainers --- chart/docs/using-additional-containers.rst | 2 +- .../pod-template-file.kubernetes-helm-yaml | 4 +- chart/templates/NOTES.txt | 8 +++ chart/values.schema.json | 18 +++++- chart/values.yaml | 10 +++ .../airflow_aux/test_pod_template_file.py | 48 ++++++++++---- .../helm_tests/airflow_core/test_worker.py | 64 +++++++++++++------ .../airflow_core/test_worker_sets.py | 14 ++++ 8 files changed, 135 insertions(+), 33 deletions(-) diff --git a/chart/docs/using-additional-containers.rst b/chart/docs/using-additional-containers.rst index 958708f1dd2ef..af339d683ea2d 100644 --- a/chart/docs/using-additional-containers.rst +++ b/chart/docs/using-additional-containers.rst @@ -52,7 +52,7 @@ Init Containers --------------- You can also deploy extra init containers through the ``extraInitContainers`` parameter. -You can define different containers for the scheduler, webserver, api server, worker, triggerer, dag processor, create user job and migrate database job pods. +You can define different containers for the scheduler, webserver/api-server, Celery/Kubernetes workers, triggerer, dag processor, create user job and migrate database job pods. For example, an init container that just says hello: diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index e9108745a82c3..d0279bd33204a 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -52,8 +52,8 @@ spec: {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }} {{- end }} - {{- if .Values.workers.extraInitContainers }} - {{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 4 }} + {{- if or .Values.workers.kubernetes.extraInitContainers .Values.workers.extraInitContainers }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraInitContainers | default .Values.workers.extraInitContainers)) . | nindent 4 }} {{- end }} {{- if or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} - name: kerberos-init diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 840e55e43ed23..654ffc527e2ff 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -709,6 +709,14 @@ DEPRECATION WARNING: {{- end }} +{{- if not (empty .Values.workers.extraInitContainers) }} + + DEPRECATION WARNING: + `workers.extraInitContainers` has been renamed to `workers.celery.extraInitContainers`/`workers.kubernetes.extraInitContainers`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.workers.runtimeClassName) }} DEPRECATION WARNING: diff --git a/chart/values.schema.json b/chart/values.schema.json index 9809be9468b43..eda22559b2377 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2301,7 +2301,7 @@ } }, "extraInitContainers": { - "description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated).", + "description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraInitContainers`` and/or ``workers.kubernetes.extraInitContainers`` instead).", "type": "array", "default": [], "items": { @@ -3388,6 +3388,14 @@ "$ref": "#/definitions/io.k8s.api.core.v1.Container" } }, + "extraInitContainers": { + "description": "Add additional init containers into Airflow Celery workers (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, "extraPorts": { "description": "Expose additional ports of Airflow Celery worker container.", "type": "array", @@ -3886,6 +3894,14 @@ "$ref": "#/definitions/io.k8s.api.core.v1.Container" } }, + "extraInitContainers": { + "description": "Add additional init containers into pods created with pod-template-file (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, "nodeSelector": { "description": "Select certain nodes for pods created with pod-template-file.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index f4b4499bac255..309b5490951a8 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1052,6 +1052,10 @@ workers: # Add additional init containers into Airflow Celery workers # and pods created with pod-template-file (templated). + # (deprecated, use + # `workers.celery.extraInitContainers` and/or + # `workers.kubernetes.extraInitContainers` + # instead) extraInitContainers: [] # Additional volumes and volume mounts attached to the @@ -1450,6 +1454,9 @@ workers: # Launch additional containers into Airflow Celery worker (templated) extraContainers: [] + # Add additional init containers into Airflow Celery workers (templated) + extraInitContainers: [] + # Expose additional ports of Airflow Celery workers. These can be used for additional metric collection. extraPorts: [] @@ -1617,6 +1624,9 @@ workers: # container finishes so Airflow can continue the worker shutdown process! extraContainers: [] + # Add additional init containers into pods created with pod-template-file (templated) + extraInitContainers: [] + # Select certain nodes for pods created with pod-template-file nodeSelector: {} diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py index 7b47e0af468f3..82d9bf2fc2c92 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -1067,15 +1067,34 @@ def test_airflow_and_workers_pod_annotations(self): assert "my_annotation" in annotations assert "workerPodAnnotations" in annotations["my_annotation"] - def test_should_add_extra_init_containers(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] + }, + { + "kubernetes": { "extraInitContainers": [ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ], + ] + } + }, + { + "extraInitContainers": [{"name": "test", "image": "repo:tag"}], + "kubernetes": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] }, }, + ], + ) + def test_should_add_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1087,13 +1106,20 @@ def test_should_add_extra_init_containers(self): } ] - def test_should_template_extra_init_containers(self): - docs = render_chart( - values={ - "workers": { - "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, + {"kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}}, + { + "extraInitContainers": [{"name": "container"}], + "kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, }, + ], + ) + def test_should_template_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index eb3286f896218..9c103af2040c9 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -279,15 +279,34 @@ def test_logs_mount_on_wait_for_migrations_initcontainer(self, logs_values, expe for m in mounts ) - def test_should_add_extra_init_containers(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] + }, + { + "celery": { "extraInitContainers": [ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ], + ] + } + }, + { + "extraInitContainers": [{"name": "container", "image": "repo:tag"}], + "celery": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] }, }, + ], + ) + def test_should_add_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -299,13 +318,20 @@ def test_should_add_extra_init_containers(self): } ] - def test_should_template_extra_init_containers(self): - docs = render_chart( - values={ - "workers": { - "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, + {"celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}}, + { + "extraInitContainers": [{"name": "container"}], + "celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, }, + ], + ) + def test_should_template_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -944,13 +970,15 @@ def test_extra_init_container_restart_policy_is_configurable(self): docs = render_chart( values={ "workers": { - "extraInitContainers": [ - { - "name": "test-init-container", - "image": "test-registry/test-repo:test-tag", - "restartPolicy": "Always", - } - ] + "celery": { + "extraInitContainers": [ + { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + "restartPolicy": "Always", + } + ] + } }, }, show_only=["templates/workers/worker-deployment.yaml"], diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py index d9ad0da07603c..898f630ff4412 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py @@ -2513,6 +2513,20 @@ def test_overwrite_extra_containers(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "extraInitContainers": [{"name": "test", "image": "test"}], + "sets": [ + { + "name": "set1", + "extraInitContainers": [ + {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} + ], + } + ], + }, + }, ], ) def test_overwrite_extra_init_containers(self, workers_values): From 56e4e3e7e5fb9a18b7a32ec414bd27f8a48aed96 Mon Sep 17 00:00:00 2001 From: Miretpl Date: Sun, 5 Apr 2026 15:24:22 +0200 Subject: [PATCH 3/3] Add newsfragment --- chart/newsfragments/64741.significant.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 chart/newsfragments/64741.significant.rst diff --git a/chart/newsfragments/64741.significant.rst b/chart/newsfragments/64741.significant.rst new file mode 100644 index 0000000000000..18a90f8f1da9c --- /dev/null +++ b/chart/newsfragments/64741.significant.rst @@ -0,0 +1 @@ +``workers.extraInitContainers`` field is now deprecated in favor of ``workers.celery.extraInitContainers`` and ``workers.kubernetes.extraInitContainers``. Please update your configuration accordingly.