wip: tls almost working

Author: adwk67

stacks/trino-iceberg/airflow.yaml

@@ -35,6 +35,8 @@ spec:
     roleGroups:
       default:
         replicas: 1
+  # kubernetesExecutors:
+  #   config: {}
   schedulers:
     roleGroups:
       default:
@@ -56,6 +58,36 @@ data:
   dbt_trino.py: |
     from airflow import DAG
     from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
+    from kubernetes.client import models as k8s
+
+    volume = k8s.V1Volume(
+        name="server-tls-mount",
+        ephemeral=k8s.V1EphemeralVolumeSource(
+            volume_claim_template=k8s.V1PersistentVolumeClaimTemplate(
+                metadata=k8s.V1ObjectMeta(
+                    annotations={
+                        "secrets.stackable.tech/class": "trino-tls",
+                        "secrets.stackable.tech/scope": "pod,node"
+                    }
+                ),
+                spec=k8s.V1PersistentVolumeClaimSpec(
+                    access_modes=["ReadWriteOnce"],
+                    resources=k8s.V1ResourceRequirements(
+                        requests={"storage": "1"}
+                    ),
+                    storage_class_name="secrets.stackable.tech"
+                )
+            )
+        )
+    )
+
+    volume_mount = k8s.V1VolumeMount(
+      name="server-tls-mount", mount_path="/dbt/trusted"
+    )
+
+    pod_security_context = k8s.V1PodSecurityContext(
+        fs_group=1000
+    )
 
     with DAG(
         dag_id="run_dbt_check_via_pod_operator",
@@ -67,10 +99,13 @@ data:
           image="my-dbt-trino:0.0.1",
           image_pull_policy="IfNotPresent",
           cmds=["/bin/bash", "-x", "-euo", "pipefail", "-c"],
-          arguments=["dbt debug"],
+          arguments=["sleep infinity #curl -v -u admin:adminadmin --cacert /dbt/trusted/ca.crt https://trino-coordinator-default-headless.default.svc.cluster.local:8443/v1/info"],
           name="run-dbt-debug",
           task_id="dbt-task",
           get_logs=True,
+          volumes=[volume],
+          volume_mounts=[volume_mount],
+          security_context=pod_security_context,
           startup_timeout_seconds=600
       )
       run_dbt_check

stacks/trino-iceberg/dbt/Dockerfile

@@ -0,0 +1,42 @@
+FROM python:3.11-slim AS builder
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create virtual environment
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install Python packages
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Final stage
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    vim \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy virtual environment from builder
+COPY --from=builder /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+WORKDIR /dbt
+
+# Copy dbt project
+COPY dbt_project.yml profiles.yml ./
+
+# Security: non-root user
+RUN useradd -m -u 1000 dbt && chown -R dbt:dbt /dbt
+USER dbt
+
+ENV DBT_PROFILES_DIR=/dbt
+
+CMD ["dbt", "run"]

stacks/trino-iceberg/dbt/dbt_project.yml

@@ -0,0 +1,21 @@
+name: 'my_dbt_project'
+version: '1.0.0'
+config-version: 2
+
+# This links to the profile name in profiles.yml
+profile: 'my_project'
+
+model-paths: ["models"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+
+clean-targets:
+  - "target"
+
+models:
+  my_dbt_project:
+    +materialized: view
+    +file_format: parquet
+
+flags:
+  require_certificate_validation: True

stacks/trino-iceberg/dbt/profiles.yml

@@ -0,0 +1,18 @@
+my_project:
+  target: prod
+  outputs:
+    prod:
+      type: trino
+      #method: none
+      user: admin
+      password: adminadmin
+      host: trino-coordinator-default-headless.default.svc.cluster.local
+      port: 8443
+      database: lakehouse
+      schema: my_schema
+      threads: 4
+      http_scheme: https
+      cert: /dbt/trusted/ca.crt
+      session_properties:
+        query_max_run_time: "4h"
+        query_priority: 1

stacks/trino-iceberg/dbt/requirements.txt

@@ -0,0 +1,3 @@
+dbt-core
+dbt-trino
+trino==0.329

stacks/trino-iceberg/trino.yaml

@@ -16,6 +16,9 @@ spec:
       opa:
         configMapName: opa
         package: trino
+    tls:
+      serverSecretClass: trino-tls
+      internalSecretClass: trino-internal-tls
   coordinators:
     roleGroups:
       default:
@@ -27,6 +30,32 @@ spec:
       default:
         replicas: 1
 ---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+  name: trino-tls
+spec:
+  backend:
+    autoTls:
+      ca:
+        secret:
+          name: secret-provisioner-trino-tls-ca
+          namespace: default
+        autoGenerate: true
+---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: SecretClass
+metadata:
+  name: trino-internal-tls
+spec:
+  backend:
+    autoTls:
+      ca:
+        secret:
+          name: secret-provisioner-trino-internal-tls-ca
+          namespace: default
+        autoGenerate: true
+---
 apiVersion: authentication.stackable.tech/v1alpha1
 kind: AuthenticationClass
 metadata:
@@ -43,7 +72,7 @@ metadata:
   name: trino-users
 type: kubernetes.io/opaque
 stringData:
-  admin: "{{ trinoAdminPassword }}"
+  admin: adminadmin
 ---
 apiVersion: trino.stackable.tech/v1alpha1
 kind: TrinoCatalog