Skip to content

Commit 0da0af2

Browse files
MiretplMiretpl
committed
Add workers.celery.extraInitContainers & workers.kubernetes.extraInitContainers
1 parent 06ecbf3 commit 0da0af2

8 files changed

Lines changed: 135 additions & 33 deletions

File tree

‎chart/docs/using-additional-containers.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ Init Containers
5050
---------------
5151

5252
You can also deploy extra init containers through the ``extraInitContainers`` parameter.
53-
You can define different containers for the scheduler, webserver, api server, worker, triggerer, dag processor, create user job and migrate database job pods.
53+
You can define different containers for the scheduler, webserver/api-server, Celery/Kubernetes workers, triggerer, dag processor, create user job and migrate database job pods.
5454

5555
For example, an init container that just says hello:
5656

‎chart/files/pod-template-file.kubernetes-helm-yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ spec:
5252
{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}
5353
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }}
5454
{{- end }}
55-
{{- if .Values.workers.extraInitContainers }}
56-
{{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 4 }}
55+
{{- if or .Values.workers.kubernetes.extraInitContainers .Values.workers.extraInitContainers }}
56+
{{- tpl (toYaml (.Values.workers.kubernetes.extraInitContainers | default .Values.workers.extraInitContainers)) . | nindent 4 }}
5757
{{- end }}
5858
{{- if or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }}
5959
- name: kerberos-init

‎chart/templates/NOTES.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,14 @@ DEPRECATION WARNING:
629629

630630
{{- end }}
631631

632+
{{- if not (empty .Values.workers.extraInitContainers) }}
633+
634+
DEPRECATION WARNING:
635+
`workers.extraInitContainers` has been renamed to `workers.celery.extraInitContainers`/`workers.kubernetes.extraInitContainers`.
636+
Please change your values as support for the old name will be dropped in a future release.
637+
638+
{{- end }}
639+
632640
{{- if not (empty .Values.workers.runtimeClassName) }}
633641

634642
DEPRECATION WARNING:

‎chart/values.schema.json‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2301,7 +2301,7 @@
23012301
}
23022302
},
23032303
"extraInitContainers": {
2304-
"description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated).",
2304+
"description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraInitContainers`` and/or ``workers.kubernetes.extraInitContainers`` instead).",
23052305
"type": "array",
23062306
"default": [],
23072307
"items": {
@@ -3294,6 +3294,14 @@
32943294
],
32953295
"default": null
32963296
},
3297+
"extraInitContainers": {
3298+
"description": "Add additional init containers into Airflow Celery workers (templated).",
3299+
"type": "array",
3300+
"default": [],
3301+
"items": {
3302+
"$ref": "#/definitions/io.k8s.api.core.v1.Container"
3303+
}
3304+
},
32973305
"extraPorts": {
32983306
"description": "Expose additional ports of Airflow Celery worker container.",
32993307
"type": "array",
@@ -3637,6 +3645,14 @@
36373645
],
36383646
"default": null
36393647
},
3648+
"extraInitContainers": {
3649+
"description": "Add additional init containers into pods created with pod-template-file (templated).",
3650+
"type": "array",
3651+
"default": [],
3652+
"items": {
3653+
"$ref": "#/definitions/io.k8s.api.core.v1.Container"
3654+
}
3655+
},
36403656
"nodeSelector": {
36413657
"description": "Select certain nodes for pods created with pod-template-file.",
36423658
"type": "object",

‎chart/values.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,6 +1025,10 @@ workers:
10251025

10261026
# Add additional init containers into Airflow Celery workers
10271027
# and pods created with pod-template-file (templated).
1028+
# (deprecated, use
1029+
# `workers.celery.extraInitContainers` and/or
1030+
# `workers.kubernetes.extraInitContainers`
1031+
# instead)
10281032
extraInitContainers: []
10291033

10301034
# Additional volumes and volume mounts attached to the
@@ -1379,6 +1383,9 @@ workers:
13791383
# This setting tells Kubernetes that its ok to evict when it wants to scale a node down
13801384
safeToEvict: ~
13811385

1386+
# Add additional init containers into Airflow Celery workers (templated)
1387+
extraInitContainers: []
1388+
13821389
# Expose additional ports of Airflow Celery workers. These can be used for additional metric collection.
13831390
extraPorts: []
13841391

@@ -1470,6 +1477,9 @@ workers:
14701477
# This setting tells Kubernetes that its ok to evict when it wants to scale a node down
14711478
safeToEvict: ~
14721479

1480+
# Add additional init containers into pods created with pod-template-file (templated)
1481+
extraInitContainers: []
1482+
14731483
# Select certain nodes for pods created with pod-template-file
14741484
nodeSelector: {}
14751485

‎helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1006,15 +1006,34 @@ def test_airflow_and_workers_pod_annotations(self):
10061006
assert "my_annotation" in annotations
10071007
assert "workerPodAnnotations" in annotations["my_annotation"]
10081008

1009-
def test_should_add_extra_init_containers(self):
1010-
docs = render_chart(
1011-
values={
1012-
"workers": {
1009+
@pytest.mark.parametrize(
1010+
"workers_values",
1011+
[
1012+
{
1013+
"extraInitContainers": [
1014+
{"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
1015+
]
1016+
},
1017+
{
1018+
"kubernetes": {
10131019
"extraInitContainers": [
10141020
{"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
1015-
],
1021+
]
1022+
}
1023+
},
1024+
{
1025+
"extraInitContainers": [{"name": "test", "image": "repo:tag"}],
1026+
"kubernetes": {
1027+
"extraInitContainers": [
1028+
{"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
1029+
]
10161030
},
10171031
},
1032+
],
1033+
)
1034+
def test_should_add_extra_init_containers(self, workers_values):
1035+
docs = render_chart(
1036+
values={"workers": workers_values},
10181037
show_only=["templates/pod-template-file.yaml"],
10191038
chart_dir=self.temp_chart_dir,
10201039
)
@@ -1026,13 +1045,20 @@ def test_should_add_extra_init_containers(self):
10261045
}
10271046
]
10281047

1029-
def test_should_template_extra_init_containers(self):
1030-
docs = render_chart(
1031-
values={
1032-
"workers": {
1033-
"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}],
1034-
},
1048+
@pytest.mark.parametrize(
1049+
"workers_values",
1050+
[
1051+
{"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]},
1052+
{"kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}},
1053+
{
1054+
"extraInitContainers": [{"name": "container"}],
1055+
"kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]},
10351056
},
1057+
],
1058+
)
1059+
def test_should_template_extra_init_containers(self, workers_values):
1060+
docs = render_chart(
1061+
values={"workers": workers_values},
10361062
show_only=["templates/pod-template-file.yaml"],
10371063
chart_dir=self.temp_chart_dir,
10381064
)

‎helm-tests/tests/helm_tests/airflow_core/test_worker.py‎

Lines changed: 46 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -261,15 +261,34 @@ def test_logs_mount_on_wait_for_migrations_initcontainer(self, logs_values, expe
261261
for m in mounts
262262
)
263263

264-
def test_should_add_extra_init_containers(self):
265-
docs = render_chart(
266-
values={
267-
"workers": {
264+
@pytest.mark.parametrize(
265+
"workers_values",
266+
[
267+
{
268+
"extraInitContainers": [
269+
{"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
270+
]
271+
},
272+
{
273+
"celery": {
268274
"extraInitContainers": [
269275
{"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
270-
],
276+
]
277+
}
278+
},
279+
{
280+
"extraInitContainers": [{"name": "container", "image": "repo:tag"}],
281+
"celery": {
282+
"extraInitContainers": [
283+
{"name": "test-init-container", "image": "test-registry/test-repo:test-tag"}
284+
]
271285
},
272286
},
287+
],
288+
)
289+
def test_should_add_extra_init_containers(self, workers_values):
290+
docs = render_chart(
291+
values={"workers": workers_values},
273292
show_only=["templates/workers/worker-deployment.yaml"],
274293
)
275294

@@ -281,13 +300,20 @@ def test_should_add_extra_init_containers(self):
281300
}
282301
]
283302

284-
def test_should_template_extra_init_containers(self):
285-
docs = render_chart(
286-
values={
287-
"workers": {
288-
"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}],
289-
},
303+
@pytest.mark.parametrize(
304+
"workers_values",
305+
[
306+
{"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]},
307+
{"celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}},
308+
{
309+
"extraInitContainers": [{"name": "container"}],
310+
"celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]},
290311
},
312+
],
313+
)
314+
def test_should_template_extra_init_containers(self, workers_values):
315+
docs = render_chart(
316+
values={"workers": workers_values},
291317
show_only=["templates/workers/worker-deployment.yaml"],
292318
)
293319

@@ -858,13 +884,15 @@ def test_extra_init_container_restart_policy_is_configurable(self):
858884
docs = render_chart(
859885
values={
860886
"workers": {
861-
"extraInitContainers": [
862-
{
863-
"name": "test-init-container",
864-
"image": "test-registry/test-repo:test-tag",
865-
"restartPolicy": "Always",
866-
}
867-
]
887+
"celery": {
888+
"extraInitContainers": [
889+
{
890+
"name": "test-init-container",
891+
"image": "test-registry/test-repo:test-tag",
892+
"restartPolicy": "Always",
893+
}
894+
]
895+
}
868896
},
869897
},
870898
show_only=["templates/workers/worker-deployment.yaml"],

‎helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2333,6 +2333,20 @@ def test_overwrite_extra_containers(self, workers_values):
23332333
],
23342334
},
23352335
},
2336+
{
2337+
"celery": {
2338+
"enableDefault": False,
2339+
"extraInitContainers": [{"name": "test", "image": "test"}],
2340+
"sets": [
2341+
{
2342+
"name": "set1",
2343+
"extraInitContainers": [
2344+
{"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"}
2345+
],
2346+
}
2347+
],
2348+
},
2349+
},
23362350
],
23372351
)
23382352
def test_overwrite_extra_init_containers(self, workers_values):

0 commit comments

Comments
 (0)