wip

Author: adwk67

demos/demos-v2.yaml

@@ -167,10 +167,6 @@ demos:
       - s3
     manifests: []
     supportedNamespaces: []
-    resourceRequests:
-      cpu: 6000m # Measured 5600m
-      memory: 21Gi
-      pvc: 110Gi # 100Gi for MinIO
   trino-taxi-data:
     description: Demo loading 2.5 years of New York taxi data into S3 bucket, creating a Trino table and a Superset dashboard
     documentation: https://docs.stackable.tech/home/stable/demos/trino-taxi-data/

stacks/stacks-v2.yaml

@@ -384,6 +384,7 @@ stacks:
       - hive
       - trino
       - opa
+      - airflow
     labels:
       - trino
       - opa
@@ -396,18 +397,22 @@ stacks:
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/s3-connection.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/hive-metastores.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/trino.yaml
-    supportedNamespaces: ["default"] # until namespace can be templates, the minio FQDN hard-codes the namespace
-    resourceRequests:
-      cpu: 6000m # Measured 5600m
-      memory: 21Gi
-      pvc: 110Gi # 100Gi for MinIO
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-airflow.yaml
+      - plainYaml: stacks/trino-iceberg/airflow.yaml
+    supportedNamespaces: ["default"]
     parameters:
       - name: trinoAdminPassword
         description: Password of the Trino admin user
         default: adminadmin
       - name: minioAdminPassword
         description: Password of the MinIO admin user
         default: adminadmin
+      - name: airflowAdminPassword
+        description: Password of the MinIO admin user
+        default: adminadmin
+      - name: airflowSecretKey
+        description: Airflow's secret key used to generate e.g. user session tokens
+        default: airflowSecretKey
   jupyterhub-pyspark-hdfs:
     description: Jupyterhub with PySpark and HDFS integration
     stackableRelease: dev

stacks/trino-iceberg/airflow.yaml

@@ -0,0 +1,57 @@
+---
+apiVersion: airflow.stackable.tech/v1alpha1
+kind: AirflowCluster
+metadata:
+  name: airflow
+spec:
+  image:
+    productVersion: 3.0.6
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    loadExamples: false
+    credentialsSecret: airflow-credentials
+  webservers:
+    roleConfig:
+      listenerClass: external-stable
+    config:
+      resources:
+        cpu:
+          min: "2"
+          max: "3"
+        memory:
+          limit: 3Gi
+    configOverrides:
+      webserver_config.py:
+        # Allow "POST /login/" without CSRF token
+        WTF_CSRF_ENABLED: "False"
+    roleGroups:
+      default:
+        replicas: 1
+  kubernetesExecutors:
+    config: {}
+  schedulers:
+    roleGroups:
+      default:
+        replicas: 1
+  dagProcessors:
+    roleGroups:
+      default:
+        replicas: 1
+  triggerers:
+    roleGroups:
+      default:
+        replicas: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: airflow-credentials
+type: Opaque
+stringData:
+  adminUser.username: admin
+  adminUser.firstname: Airflow
+  adminUser.lastname: Admin
+  adminUser.email: airflow@airflow.com
+  adminUser.password: "adminadmin"
+  connections.secretKey: "airflowSecretKey"
+  connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow

stacks/trino-iceberg/trino.yaml

@@ -17,27 +17,12 @@ spec:
         configMapName: opa
         package: trino
   coordinators:
-    config:
-      queryMaxMemory: 10TB
-      resources:
-        cpu:
-          min: "500m"
-          max: "2"
-        memory:
-          limit: '4Gi'
     roleGroups:
       default:
         replicas: 1
     roleConfig:
       listenerClass: external-stable
   workers:
-    config:
-      resources:
-        cpu:
-          min: "1"
-          max: "4"
-        memory:
-          limit: '8Gi'
     roleGroups:
       default:
         replicas: 1