Skip to content

Commit 33d0d37

Browse files
NickLarsenNZNickLarsenNZ
authored
fix(airflow): Update manifests (#431)
* fix(airflow): Update manifests - `credentialsSecret` -> `credentialsSecretName` - Change admin secret name - Add `metadataDatabase.postgresql`, add new secret - Move DB secrets - Remove legacy secret key * fix(airflow): Use correct postgres service name * docs(demo/airflow-scheduled-job): Add missing namespace to command * fix(stack/airflow): Get the Trino namespace from the env The previous templating interpolation was inside a `{% raw %}` block.
1 parent 6560ea9 commit 33d0d37

6 files changed

Lines changed: 56 additions & 26 deletions

File tree

‎demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,16 @@ spec:
2323
- |
2424
kubectl rollout status --watch statefulset/airflow-webserver-default
2525
kubectl rollout status --watch statefulset/airflow-scheduler-default
26-
AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
26+
AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-admin-credentials/adminUser.password)
2727
ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token)
2828
curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag -d '{"is_paused": false}' | jq
2929
curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag/dagRuns -d '{"logical_date": null}' | jq
3030
volumeMounts:
31-
- name: airflow-credentials
32-
mountPath: /airflow-credentials
31+
- name: airflow-admin-credentials
32+
mountPath: /airflow-admin-credentials
3333
volumes:
34-
- name: airflow-credentials
34+
- name: airflow-admin-credentials
3535
secret:
36-
secretName: airflow-credentials
36+
secretName: airflow-admin-credentials
3737
restartPolicy: OnFailure
3838
backoffLimit: 20 # give some time for the Airflow cluster to be available

‎demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,16 @@ spec:
2323
- |
2424
kubectl rollout status --watch statefulset/airflow-webserver-default
2525
kubectl rollout status --watch statefulset/airflow-scheduler-default
26-
AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
26+
AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-admin-credentials/adminUser.password)
2727
ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token)
2828
curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo -d '{"is_paused": false}' | jq
2929
curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo/dagRuns -d '{"logical_date": null}' | jq
3030
volumeMounts:
31-
- name: airflow-credentials
32-
mountPath: /airflow-credentials
31+
- name: airflow-admin-credentials
32+
mountPath: /airflow-admin-credentials
3333
volumes:
34-
- name: airflow-credentials
34+
- name: airflow-admin-credentials
3535
secret:
36-
secretName: airflow-credentials
36+
secretName: airflow-admin-credentials
3737
restartPolicy: OnFailure
3838
backoffLimit: 20 # give some time for the Airflow cluster to be available

‎demos/airflow-scheduled-job/05-enable-and-run-kafka-dag.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ spec:
4747
kubectl rollout status --watch statefulset/airflow-scheduler-default
4848
4949
# Airflow: activate DAG
50-
AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
50+
AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-admin-credentials/adminUser.password)
5151
ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token)
5252
curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/kafka_watcher -d '{"is_paused": false}' | jq
5353
@@ -58,11 +58,11 @@ spec:
5858
--producer.config /stackable/config/client.properties \
5959
--topic test-topic'
6060
volumeMounts:
61-
- name: airflow-credentials
62-
mountPath: /airflow-credentials
61+
- name: airflow-admin-credentials
62+
mountPath: /airflow-admin-credentials
6363
volumes:
64-
- name: airflow-credentials
64+
- name: airflow-admin-credentials
6565
secret:
66-
secretName: airflow-credentials
66+
secretName: airflow-admin-credentials
6767
restartPolicy: OnFailure
6868
backoffLimit: 20 # give some time for the Airflow cluster to be available

‎demos/airflow-scheduled-job/06-create-opa-users.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ spec:
4242
--password "richard.roe" \
4343
--role "User"
4444
volumeMounts:
45-
- name: airflow-credentials
46-
mountPath: /airflow-credentials
45+
- name: airflow-admin-credentials
46+
mountPath: /airflow-admin-credentials
4747
volumes:
48-
- name: airflow-credentials
48+
- name: airflow-admin-credentials
4949
secret:
50-
secretName: airflow-credentials
50+
secretName: airflow-admin-credentials
5151
restartPolicy: OnFailure
5252
backoffLimit: 20 # give some time for the Airflow cluster to be available

‎docs/modules/demos/pages/airflow-scheduled-job.adoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ To list the installed Stackable services run the following command:
5757

5858
[source,console]
5959
----
60-
$ stackablectl stacklet list
60+
$ stackablectl stacklet list -n airflow-demo
6161
6262
┌─────────┬───────────────┬───────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐
6363
│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │

‎stacks/airflow/airflow.yaml‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ spec:
99
productVersion: 3.2.2
1010
pullPolicy: IfNotPresent
1111
clusterConfig:
12+
metadataDatabase:
13+
postgresql:
14+
host: postgresql-airflow
15+
database: airflow
16+
credentialsSecretName: airflow-postgresql-credentials
1217
authorization:
1318
opa:
1419
configMapName: opa
@@ -17,7 +22,7 @@ spec:
1722
entryTimeToLive: 5s
1823
maxEntries: 10
1924
loadExamples: false
20-
credentialsSecret: airflow-credentials
25+
credentialsSecretName: airflow-admin-credentials
2126
volumes:
2227
- name: airflow-dags
2328
configMap:
@@ -105,6 +110,11 @@ spec:
105110
image: oci.stackable.tech/sdp/airflow:3.1.6-stackable0.0.0-dev
106111
imagePullPolicy: IfNotPresent
107112
env:
113+
# Needed by the DAGs run by KubernetesPodOperator
114+
- name: NAMESPACE
115+
valueFrom:
116+
fieldRef:
117+
fieldPath: metadata.namespace
108118
- name: AWS_SECRET_ACCESS_KEY
109119
valueFrom:
110120
secretKeyRef:
@@ -140,7 +150,14 @@ data:
140150
from airflow import DAG
141151
from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
142152
from kubernetes.client import models as k8s
143-
from kubernetes.client import V1EnvVar, V1EnvVarSource, V1SecretKeySelector
153+
from kubernetes.client import V1EnvVar, V1EnvVarSource, V1ObjectFieldSelector, V1SecretKeySelector
154+
import os
155+
156+
# NOTE: This code is executed on the Pod launched by the KubernetesPodOperator
157+
# Therefore, the NAMESPACE env must be set on that.
158+
# See AirflowCluster.spec.kubernetesExecutor.podOverrides.
159+
namespace = os.environ.get("NAMESPACE", "default")
160+
trino_coordinator_hostname = f"trino-coordinator-default-headless.{namespace}.svc.cluster.local"
144161
145162
tls_volume = k8s.V1Volume(
146163
name="server-tls-mount",
@@ -188,6 +205,12 @@ data:
188205
volumes=[tls_volume],
189206
volume_mounts=[tls_volume_mount],
190207
env_vars=[
208+
V1EnvVar(
209+
name="NAMESPACE",
210+
value_from=V1EnvVarSource(
211+
field_ref=V1ObjectFieldSelector(field_path="metadata.namespace")
212+
)
213+
),
191214
V1EnvVar(
192215
name="TRINO_PASSWORD",
193216
value_from=V1EnvVarSource(
@@ -198,7 +221,7 @@ data:
198221
)
199222
),
200223
V1EnvVar(name="TRINO_USER", value="admin"),
201-
V1EnvVar(name="TRINO_HOST", value="trino-coordinator-default-headless.{{ NAMESPACE }}.svc.cluster.local"),
224+
V1EnvVar(name="TRINO_HOST", value=trino_coordinator_hostname),
202225
V1EnvVar(name="TRINO_PORT", value="8443"),
203226
V1EnvVar(name="CERT_PATH", value="/dbt/trusted/ca.crt"),
204227
],
@@ -571,13 +594,20 @@ data:
571594
apiVersion: v1
572595
kind: Secret
573596
metadata:
574-
name: airflow-credentials
597+
name: airflow-admin-credentials
575598
type: Opaque
576599
stringData:
577600
adminUser.username: admin
578601
adminUser.firstname: Airflow
579602
adminUser.lastname: Admin
580603
adminUser.email: airflow@airflow.com
581604
adminUser.password: "{{ airflowAdminPassword }}"
582-
connections.secretKey: "{{ airflowSecretKey }}"
583-
connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow
605+
---
606+
apiVersion: v1
607+
kind: Secret
608+
metadata:
609+
name: airflow-postgresql-credentials
610+
type: Opaque
611+
stringData:
612+
username: airflow
613+
password: airflow

0 commit comments

Comments
 (0)