fix(airflow): Update manifests

- `credentialsSecret` -> `credentialsSecretName`
- Change admin secret name
- Add `metadataDatabase.postgresql`, add new secret
- Move DB secrets
- Remove legacy secret key

Author: NickLarsenNZ

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -23,16 +23,16 @@ spec:
           - |
               kubectl rollout status --watch statefulset/airflow-webserver-default
               kubectl rollout status --watch statefulset/airflow-scheduler-default
-              AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
+              AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-admin-credentials/adminUser.password)
               ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token)
               curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag -d '{"is_paused": false}' | jq
               curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag/dagRuns -d '{"logical_date": null}' | jq
           volumeMounts:
-            - name: airflow-credentials
-              mountPath: /airflow-credentials
+            - name: airflow-admin-credentials
+              mountPath: /airflow-admin-credentials
       volumes:
-        - name: airflow-credentials
+        - name: airflow-admin-credentials
           secret:
-            secretName: airflow-credentials
+            secretName: airflow-admin-credentials
       restartPolicy: OnFailure
   backoffLimit: 20 # give some time for the Airflow cluster to be available

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -23,16 +23,16 @@ spec:
           - |
               kubectl rollout status --watch statefulset/airflow-webserver-default
               kubectl rollout status --watch statefulset/airflow-scheduler-default
-              AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
+              AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-admin-credentials/adminUser.password)
               ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token)
               curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo -d '{"is_paused": false}' | jq
               curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo/dagRuns -d '{"logical_date": null}' | jq
           volumeMounts:
-            - name: airflow-credentials
-              mountPath: /airflow-credentials
+            - name: airflow-admin-credentials
+              mountPath: /airflow-admin-credentials
       volumes:
-        - name: airflow-credentials
+        - name: airflow-admin-credentials
           secret:
-            secretName: airflow-credentials
+            secretName: airflow-admin-credentials
       restartPolicy: OnFailure
   backoffLimit: 20 # give some time for the Airflow cluster to be available

demos/airflow-scheduled-job/05-enable-and-run-kafka-dag.yaml

@@ -47,7 +47,7 @@ spec:
               kubectl rollout status --watch statefulset/airflow-scheduler-default
 
               # Airflow: activate DAG
-              AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password)
+              AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-admin-credentials/adminUser.password)
               ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token)
               curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/kafka_watcher -d '{"is_paused": false}' | jq
 
@@ -58,11 +58,11 @@ spec:
                 --producer.config /stackable/config/client.properties \
                 --topic test-topic'
           volumeMounts:
-            - name: airflow-credentials
-              mountPath: /airflow-credentials
+            - name: airflow-admin-credentials
+              mountPath: /airflow-admin-credentials
       volumes:
-        - name: airflow-credentials
+        - name: airflow-admin-credentials
           secret:
-            secretName: airflow-credentials
+            secretName: airflow-admin-credentials
       restartPolicy: OnFailure
   backoffLimit: 20 # give some time for the Airflow cluster to be available

demos/airflow-scheduled-job/06-create-opa-users.yaml

@@ -42,11 +42,11 @@ spec:
                 --password "richard.roe" \
                 --role "User"
           volumeMounts:
-            - name: airflow-credentials
-              mountPath: /airflow-credentials
+            - name: airflow-admin-credentials
+              mountPath: /airflow-admin-credentials
       volumes:
-        - name: airflow-credentials
+        - name: airflow-admin-credentials
           secret:
-            secretName: airflow-credentials
+            secretName: airflow-admin-credentials
       restartPolicy: OnFailure
   backoffLimit: 20 # give some time for the Airflow cluster to be available

stacks/airflow/airflow.yaml

@@ -9,6 +9,11 @@ spec:
     productVersion: 3.2.2
     pullPolicy: IfNotPresent
   clusterConfig:
+    metadataDatabase:
+      postgresql:
+        host: airflow-postgresql
+        database: airflow
+        credentialsSecretName: airflow-postgresql-credentials
     authorization:
       opa:
         configMapName: opa
@@ -17,7 +22,7 @@ spec:
           entryTimeToLive: 5s
           maxEntries: 10
     loadExamples: false
-    credentialsSecret: airflow-credentials
+    credentialsSecretName: airflow-admin-credentials
     volumes:
       - name: airflow-dags
         configMap:
@@ -571,13 +576,20 @@ data:
 apiVersion: v1
 kind: Secret
 metadata:
-  name: airflow-credentials
+  name: airflow-admin-credentials
 type: Opaque
 stringData:
   adminUser.username: admin
   adminUser.firstname: Airflow
   adminUser.lastname: Admin
   adminUser.email: airflow@airflow.com
   adminUser.password: "{{ airflowAdminPassword }}"
-  connections.secretKey: "{{ airflowSecretKey }}"
-  connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: airflow-postgresql-credentials
+type: Opaque
+stringData:
+  username: airflow
+  password: airflow