wip

Author: maltesander

demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml

@@ -10,7 +10,14 @@ spec:
       initContainers:
         - name: wait-for-kafka
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for all kafka brokers to be ready' 
+              kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka
       containers:
         - name: create-nifi-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
@@ -54,7 +61,7 @@ data:
     import urllib3
 
     # As of 2022-08-29 we cant use "https://nifi:8443" here because <h2>The request contained an invalid host header [<code>nifi:8443</code>] in the request [<code>/nifi-api</code>]. Check for request manipulation or third-party intercept.</h2>
-    ENDPOINT = f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443"
+    ENDPOINT = f"https://nifi-node-default-0.nifi-node-default-headless.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default-headless.{os.environ['NAMESPACE']}.svc.cluster.local:8443"
     USERNAME = "admin"
     PASSWORD = open("/nifi-admin-credentials-secret/admin").read()
 

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -13,11 +13,24 @@ spec:
       initContainers:
         - name: wait-for-kafka
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for all kafka brokers to be ready'
+              kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka
       containers:
         - name: create-spark-ingestion-job
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml
           volumeMounts:
             - name: manifest
               mountPath: /tmp/manifest
@@ -56,7 +69,7 @@ data:
         spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
         spark.sql.catalog.lakehouse: org.apache.iceberg.spark.SparkCatalog
         spark.sql.catalog.lakehouse.type: hive
-        spark.sql.catalog.lakehouse.uri: thrift://hive-iceberg:9083
+        spark.sql.catalog.lakehouse.uri: thrift://hive-iceberg-metastore:9083
         # Every merge into statements creates 8 files.
         # Paralleling is enough for the demo, might need to be increased (or omitted entirely) when merge larger data volumes
         spark.sql.shuffle.partitions: "8"

demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml

@@ -10,7 +10,14 @@ spec:
       initContainers:
         - name: wait-for-testdata
           image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for job load-test-data to finish'
+              kubectl wait --for=condition=complete --timeout=30m job/load-test-data
       containers:
         - name: create-tables-in-trino
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -9,7 +9,11 @@ spec:
       containers:
         - name: setup-superset
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
+          command:
+            - "bash"
+            - "-c"
+            - "pipefail"
+            - "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"
           volumeMounts:
             - name: script
               mountPath: /tmp/script
@@ -39,7 +43,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
+    base_url = "http://superset-node:8088" # For local testing / developing replace it, afterwards change back to http://superset-node:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"

stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml

@@ -9,18 +9,19 @@ spec:
   clusterConfig:
     authentication:
       - authenticationClass: nifi-admin-credentials
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     config:
       resources:
         cpu:
           min: "2"
           max: "4"
         memory:
-          limit: '6Gi'
+          limit: "6Gi"
         storage:
           contentRepo:
             capacity: "10Gi"
@@ -51,4 +52,4 @@ kind: Secret
 metadata:
   name: nifi-admin-credentials-secret
 stringData:
-  admin: {{ nifiAdminPassword }}
+  admin: "{{ nifiAdminPassword }}"

stacks/data-lakehouse-iceberg-trino-spark/trino.yaml

@@ -7,7 +7,6 @@ spec:
   image:
     productVersion: "476"
   clusterConfig:
-    listenerClass: external-unstable
     catalogLabelSelector:
       matchLabels:
         trino: trino
@@ -18,14 +17,16 @@ spec:
         configMapName: opa
         package: trino
   coordinators:
+    roleConfig:
+      listenerClass: external-unstable
     config:
       queryMaxMemory: 10TB
       resources:
         cpu:
           min: "1"
           max: "4"
         memory:
-          limit: '6Gi'
+          limit: "6Gi"
     roleGroups:
       default:
         replicas: 1
@@ -37,7 +38,7 @@ spec:
           min: "2"
           max: "6"
         memory:
-          limit: '20Gi'
+          limit: "20Gi"
     roleGroups:
       default:
         replicas: 4