wip

Author: maltesander

demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml

@@ -9,11 +9,21 @@ spec:
       initContainers:
         - name: wait-for-testdata
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data"]
+          command:
+            [
+              "bash",
+              "-c",
+              "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data",
+            ]
       containers:
         - name: create-spark-anomaly-detection-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml"]
+          command:
+            [
+              "bash",
+              "-c",
+              "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml",
+            ]
           volumeMounts:
             - name: manifest
               mountPath: /tmp/manifest
@@ -89,7 +99,7 @@ data:
         spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
         spark.sql.catalog.prediction: org.apache.iceberg.spark.SparkCatalog
         spark.sql.catalog.prediction.type: hive
-        spark.sql.catalog.prediction.uri: thrift://hive-iceberg:9083
+        spark.sql.catalog.prediction.uri: thrift://hive-iceberg-metastore:9083
         # AFAIK This can be removed in 24.3
         #spark.driver.userClassPathFirst: "false"
         #spark.executor.userClassPathFirst: "false"

demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml

@@ -9,7 +9,12 @@ spec:
       containers:
         - name: setup-superset
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
+          command:
+            [
+              "bash",
+              "-c",
+              "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py",
+            ]
           volumeMounts:
             - name: script
               mountPath: /tmp/script
@@ -39,7 +44,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
+    base_url = "http://superset-node-default-headless:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default-headless:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"

demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip

@@ -5,7 +5,7 @@ metadata:
   name: superset
 spec:
   image:
-    productVersion: 4.1.2
+    productVersion: 4.0.2
   clusterConfig:
     credentialsSecret: superset-credentials
     mapboxSecret: superset-mapbox-api-key
@@ -26,8 +26,8 @@ stringData:
   adminUser.firstname: Superset
   adminUser.lastname: Admin
   adminUser.email: admin@superset.com
-  adminUser.password: {{ supersetAdminPassword }}
-  connections.secretKey: {{ supersetSecretKey }}
+  adminUser.password: "{{ supersetAdminPassword }}"
+  connections.secretKey: "{{ supersetSecretKey }}"
   connections.sqlalchemyDatabaseUri: postgresql://superset:superset@postgresql-superset/superset
 ---
 apiVersion: v1