fix(signal-processing): Correctly template namespace in notebook (#437)

* fix(signal-processing): Correctly template namespace in notebook

* temp: Fetch notebook from branch with fixes

* fix(signal-processing): Use different placeholder in NiFi flow

* temp: Fetch NiFi flow from branch with fixes

* chore(stack/signal-processing): Add some guidance for finding Helm values

* chore(stack/signal-processing): Disable cloudMetadata block

The iptables call ing the It fails for me. I am on kernel 7.0. It could be that.
Or maybe oci.stackable.tech/demos/jupyter-pyspark-with-alibi-detect needs some update?

* fix(stack/signal-processing): Namespace interpolation

Add extraEnv for providing the templated namespace.
We would use the downward api, but that breaks because the value of extraEnv is also passed to `hub` which expects a dict.

* chore: Apply suggestions

Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com>

---------

Co-authored-by: Nick Larsen <nick.larsen@stackable.tech>
Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com>

Author: Techassi

demos/signal-processing/DownloadAndWriteToDB.json

@@ -741,7 +741,7 @@
           "kerberos-credentials-service": null,
           "dbcp-max-conn-lifetime": "-1",
           "Validation-query": "select count(*) from conditions;",
-          "Database Connection URL": "jdbc:postgresql://postgresql-timescaledb.{{ NAMESPACE }}.svc.cluster.local:5432/tsdb",
+          "Database Connection URL": "jdbc:postgresql://postgresql-timescaledb.[[ NAMESPACE ]].svc.cluster.local:5432/tsdb",
           "dbcp-time-between-eviction-runs": "-1",
           "Database User": "admin",
           "kerberos-user-service": null,

demos/signal-processing/create-nifi-ingestion-job.yaml

@@ -90,11 +90,11 @@ data:
     # At release time, .scripts/update_refs.sh rewrites the branch in this URL to the release
     # branch. Keep the URL in a form its regex matches, or release-branch installs will
     # download this file from main (see the 26.3 regression).
-    response = requests.get("https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.json")
+    response = requests.get("https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/demos/signal-processing/DownloadAndWriteToDB.json")
 
     filename = "/tmp/DownloadAndWriteToDB.json"
     with open(filename, "w") as f:
-        f.write(response.text.replace("{{ NAMESPACE }}", os.environ["NAMESPACE"]))
+        f.write(response.text.replace("[[ NAMESPACE ]]", os.environ["NAMESPACE"]))
 
     pg_id = get_root_pg_id()
 

stacks/signal-processing/jupyterhub.yaml

@@ -5,6 +5,8 @@ repo:
   name: jupyterhub
   url: https://jupyterhub.github.io/helm-chart/
 version: 4.4.0 # appVersion: 5.5.0
+# Find values here: https://github.com/jupyterhub/zero-to-jupyterhub-k8s/blob/main/jupyterhub/values.yaml
+# Note, they don't tag the repository, so you need to find _a_ commit where the Chart version matches what you are looking for.
 options:
   hub:
     config:
@@ -40,6 +42,11 @@ options:
       # the demo is reproducable for the release and it will be automatically replaced for the release branch.
       name: oci.stackable.tech/demos/jupyter-pyspark-with-alibi-detect
       tag: python-3.9
+    cloudMetadata:
+      # Disable, otherwise we get Back-off restarting failed container block-cloud-metadata in pod:
+      # block-cloud-metadata iptables v1.8.9 (legacy): can't initialize iptables table `filter': Table does not exist (do you need to insmod?)
+      # I believe this breaks in Kernel 7.x.
+      blockWithIptables: false
     serviceAccountName: spark
     networkPolicy:
       enabled: false
@@ -51,11 +58,19 @@ options:
       # {% if DEMO is defined %}
       stackable.tech/demo: "{{ DEMO }}"
       # {% endif %}
+    extraEnv:
+      # Needed because the Notebook runs code to interpolate the Postgres DB hostname with a namespace.
+      # We cannot use the downward API because some python code uses it and expects scalar values.
+      # We also MUST use a dict, otherwise hub shows the following error:
+      # traitlets.traitlets.TraitError: The 'environment' trait of a KubeSpawner instance expected a dict, not the list [{'name': 'NAMESPACE', 'value': 'my-namespace'}].
+      # A helper in the chart formats these properly for any container env vars in Pods.
+      NAMESPACE: "{{ NAMESPACE }}"
     initContainers:
       - name: download-notebook
         image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+        env:
         # This file path needs adjustment for versioned demos on release branch
-        command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/signal-processing/tsdb.ipynb -o /notebook/tsdb.ipynb']
+        command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/refs/heads/main/stacks/signal-processing/tsdb.ipynb -o /notebook/tsdb.ipynb']
         volumeMounts:
           - mountPath: /notebook
             name: notebook

stacks/signal-processing/tsdb.ipynb

@@ -68,7 +68,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "engine = sqlalchemy.create_engine(\"postgresql://admin:adminadmin@postgresql-timescaledb.{{ NAMESPACE }}.svc.cluster.local:5432/tsdb\")\n",
+    "import os\n",
+    "namespace = os.environ.get('NAMESPACE', \"default\")\n",
+    "\n",
+    "engine = sqlalchemy.create_engine(f\"postgresql://admin:adminadmin@postgresql-timescaledb.{namespace}.svc.cluster.local:5432/tsdb\")\n",
     "last_time = \"1970-01-01\"\n",
     "cols = ['r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14']"
    ]