refactor(e2e): data-driven per-tool matrix (replaces scenario suite)

charts/e2e-infra/.gitignore

@@ -0,0 +1,3 @@
+charts/
+Chart.lock
+*.tgz

charts/e2e-infra/Chart.yaml

@@ -0,0 +1,18 @@
+apiVersion: v2
+name: e2e-infra
+description: >-
+  Self-contained E2E test infrastructure for underpass-runtime. Deploys every
+  backend the per-tool matrix needs (redis, mongo, rabbitmq, kafka, prometheus),
+  an internal git server serving the polyglot fixture repo, HTTP/notify sinks,
+  and k8s rollout/saturation target deployments — plus a dedicated test runtime
+  wired to all of them. No dependency on any pre-existing cluster service:
+  `helm install` brings it all up, `helm uninstall` tears it all down.
+type: application
+version: 0.1.0
+appVersion: "1.0.0"
+
+dependencies:
+  - name: underpass-runtime
+    version: "0.1.0"
+    repository: "file://../underpass-runtime"
+    alias: runtime

charts/e2e-infra/templates/_helpers.tpl

@@ -0,0 +1,6 @@
+{{/* Common labels for every e2e-infra object. */}}
+{{- define "e2e-infra.labels" -}}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/part-of: e2e-infra
+{{- end -}}

charts/e2e-infra/templates/backends.yaml

@@ -0,0 +1,100 @@
+{{- range .Values.backends }}
+{{- if .image }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: e2e-{{ .name }}
+  labels:
+    {{- include "e2e-infra.labels" $ | nindent 4 }}
+    app: e2e-{{ .name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: e2e-{{ .name }}
+  template:
+    metadata:
+      labels:
+        {{- include "e2e-infra.labels" $ | nindent 8 }}
+        app: e2e-{{ .name }}
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: {{ .name }}
+          image: {{ .image }}
+          resources:
+            limits:
+              memory: {{ .memoryLimit | default "1Gi" }}
+            requests:
+              memory: {{ .memoryRequest | default "128Mi" }}
+          {{- with .args }}
+          args: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - containerPort: {{ .port }}
+            {{- range .extraPorts }}
+            - containerPort: {{ . }}
+            {{- end }}
+          {{- with .env }}
+          env: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          {{- if .configMount }}
+          volumeMounts:
+            - name: config
+              mountPath: {{ .configMount }}
+              subPath: {{ .configKey }}
+          {{- end }}
+          readinessProbe:
+            {{- if .readyExec }}
+            exec:
+              command: {{ toYaml .readyExec | nindent 16 }}
+            {{- else if .readyHttp }}
+            httpGet:
+              path: {{ .readyHttp }}
+              port: {{ .port }}
+            {{- else }}
+            tcpSocket:
+              port: {{ .port }}
+            {{- end }}
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 30
+      {{- if .configMount }}
+      volumes:
+        - name: config
+          configMap:
+            name: e2e-{{ .name }}-config
+      {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: e2e-{{ .name }}
+  labels:
+    {{- include "e2e-infra.labels" $ | nindent 4 }}
+    app: e2e-{{ .name }}
+spec:
+  selector:
+    app: e2e-{{ .name }}
+  ports:
+    - name: main
+      port: {{ .port }}
+      targetPort: {{ .port }}
+    {{- range $i, $p := .extraPorts }}
+    - name: extra-{{ $i }}
+      port: {{ $p }}
+      targetPort: {{ $p }}
+    {{- end }}
+{{- end }}
+{{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: e2e-prometheus-config
+  labels:
+    {{- include "e2e-infra.labels" . | nindent 4 }}
+data:
+  prometheus.yml: |
+    {{- .Values.prometheusConfig | nindent 4 }}

charts/e2e-infra/templates/e2e-test-job.yaml

@@ -0,0 +1,51 @@
+{{- /* `helm test e2e` runs the per-tool matrix against the dedicated test
+       runtime, using the in-cluster git fixture and backends. Everything it
+       needs was installed by this same release — no external dependency. */ -}}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ .Release.Name }}-tool-matrix
+  labels:
+    {{- include "e2e-infra.labels" . | nindent 4 }}
+    app: e2e-tool-matrix
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  restartPolicy: Never
+  imagePullSecrets:
+    - name: {{ .Values.imagePullSecret | default "ghcr-pull" }}
+  containers:
+    - name: tool-matrix
+      image: {{ .Values.registry }}/e2e-tool-matrix:{{ .Values.toolMatrixTag | default "v1.0.0" }}
+      imagePullPolicy: Always
+      env:
+        # Plain gRPC to the test runtime (tls.mode=disabled).
+        - name: WORKSPACE_URL
+          value: "e2e-runtime:{{ .Values.runtime.service.port | default 50053 }}"
+        # Clone the in-cluster fixture (no github dependency).
+        - name: FIXTURE_REPO_URL
+          value: "git://e2e-gitfixture:9418/polyglot"
+        - name: FIXTURE_REPO_REF
+          value: "main"
+        - name: WORKSPACE_AUTH_TENANT_ID
+          value: "e2e-tenant"
+        - name: WORKSPACE_AUTH_ACTOR_ID
+          value: "e2e-actor"
+        - name: WORKSPACE_AUTH_ROLES
+          value: "developer,devops,platform_admin"
+        - name: CATALOG_PATH
+          value: "/app/catalog_defaults.yaml"
+        - name: SPECS_DIR
+          value: "/app/specs"
+        - name: EVIDENCE_FILE
+          value: "/tmp/evidence-tool-matrix.json"
+        - name: PYTHONUNBUFFERED
+          value: "1"
+      resources:
+        requests:
+          cpu: "200m"
+          memory: "256Mi"
+        limits:
+          cpu: "1000m"
+          memory: "512Mi"

charts/e2e-infra/templates/k8s-targets.yaml

@@ -0,0 +1,47 @@
+{{- /* Target Deployments so k8s.* and rollout/saturation tools have real
+       objects to inspect and mutate inside the release namespace. */ -}}
+{{- range .Values.k8sTargets }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .name }}
+  labels:
+    {{- include "e2e-infra.labels" $ | nindent 4 }}
+    app: {{ .name }}
+spec:
+  replicas: {{ .replicas }}
+  selector:
+    matchLabels:
+      app: {{ .name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .name }}
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: app
+          image: {{ .image }}
+          resources:
+            limits:
+              memory: {{ .memoryLimit | default "256Mi" }}
+            requests:
+              memory: {{ .memoryRequest | default "64Mi" }}
+          ports:
+            - containerPort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .name }}
+  labels:
+    {{- include "e2e-infra.labels" $ | nindent 4 }}
+    app: {{ .name }}
+spec:
+  selector:
+    app: {{ .name }}
+  ports:
+    - port: 80
+      targetPort: 80
+{{- end }}

charts/e2e-infra/values.yaml

@@ -0,0 +1,116 @@
+# ── e2e-infra values ────────────────────────────────────────────────────────
+# Self-contained test infrastructure. Every component is namespaced under this
+# release; nothing references a pre-existing cluster service.
+
+nameOverride: e2e
+
+# Registry for the images this chart builds (git fixture). Backends use upstream
+# public images so the suite stays dependency-free.
+registry: ghcr.io/underpass-ai/underpass-runtime
+
+# ── Stateful/stateless backends ─────────────────────────────────────────────
+# Generic deployment+service per backend (templates/backends.yaml). Each entry
+# becomes Deployment `e2e-<name>` + Service `e2e-<name>` on the listed port.
+backends:
+  - name: redis
+    image: docker.io/library/redis:7-alpine
+    port: 6379
+    readyExec: ["redis-cli", "ping"]
+
+  - name: mongo
+    image: docker.io/library/mongo:7
+    port: 27017
+    readyTcp: true
+
+  - name: rabbit
+    image: docker.io/library/rabbitmq:3.13-management
+    port: 5672
+    extraPorts: [15672]
+    readyExec: ["rabbitmq-diagnostics", "-q", "ping"]
+
+  - name: kafka
+    image: docker.io/apache/kafka:3.8.0   # KRaft single-node, no zookeeper
+    port: 9092
+    readyTcp: true
+    env:
+      - { name: KAFKA_NODE_ID, value: "1" }
+      - { name: KAFKA_PROCESS_ROLES, value: "broker,controller" }
+      - { name: KAFKA_LISTENERS, value: "PLAINTEXT://:9092,CONTROLLER://:9093" }
+      - { name: KAFKA_ADVERTISED_LISTENERS, value: "PLAINTEXT://e2e-kafka:9092" }
+      - { name: KAFKA_CONTROLLER_LISTENER_NAMES, value: "CONTROLLER" }
+      - { name: KAFKA_CONTROLLER_QUORUM_VOTERS, value: "1@localhost:9093" }
+      - { name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, value: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" }
+      - { name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR, value: "1" }
+      - { name: KAFKA_AUTO_CREATE_TOPICS_ENABLE, value: "true" }
+
+  - name: prometheus
+    image: docker.io/prom/prometheus:v2.54.1
+    port: 9090
+    readyHttp: /-/ready
+    configMount: /etc/prometheus/prometheus.yml
+    configKey: prometheus.yml
+
+  - name: httpbin            # target for web.fetch / api.benchmark
+    image: docker.io/kennethreitz/httpbin:latest
+    port: 80
+    readyHttp: /get
+
+  - name: notify-sink        # target for notify.escalation_channel
+    image: docker.io/kennethreitz/httpbin:latest
+    port: 80
+    readyHttp: /get
+
+  - name: gitfixture         # internal git server with the polyglot fixture repo
+    image: ghcr.io/underpass-ai/underpass-runtime/e2e-gitfixture:v1.0.0
+    port: 9418               # git daemon — repo_url = git://e2e-gitfixture:9418/polyglot
+    readyTcp: true
+
+# ── k8s target deployments (for k8s.* + rollout/saturation tools) ───────────
+k8sTargets:
+  - name: web-api
+    image: docker.io/library/nginx:1.27-alpine
+    replicas: 2
+  - name: worker
+    image: docker.io/library/nginx:1.27-alpine
+    replicas: 1
+
+# Prometheus scrape config (minimal — scrapes itself so query has data).
+prometheusConfig: |
+  global:
+    scrape_interval: 5s
+  scrape_configs:
+    - job_name: prometheus
+      static_configs:
+        - targets: ["localhost:9090"]
+
+# ── Dedicated test runtime (subchart `runtime`) ─────────────────────────────
+# Wired to the backends above. Endpoints are stable in-namespace DNS names.
+runtime:
+  config:
+    workspaceBackend: kubernetes
+    # Connection-profile endpoints the runtime resolves for db/queue tools.
+    connProfileEndpointsJSON: >-
+      {"dev.redis":"e2e-redis:6379",
+       "dev.mongo":"mongodb://e2e-mongo:27017",
+       "dev.rabbit":"amqp://e2e-rabbit:5672",
+       "dev.kafka":"e2e-kafka:9092",
+       "dev.nats":"nats://e2e-runtime-nats:4222"}
+    # notify.escalation_channel route for environment "e2e".
+    notifyEscalationRoutesJSON: '{"e2e":"http://e2e-notify-sink/post"}'
+  kubernetesBackend:
+    rbac:
+      create: true
+    deliveryTools:
+      enabled: true
+    namespace: ""   # defaults to release namespace
+    serviceAccount: ""
+    # The fat runner has every toolchain + trivy/syft/k6, so all sessions can
+    # execute every tool family. A session may still request a leaner bundle via
+    # metadata.runner_profile.
+    runnerImage: ghcr.io/underpass-ai/underpass-runtime/workspace-runner-fat:v1.0.0
+    imageBundlesJSON: >-
+      {"fat":"ghcr.io/underpass-ai/underpass-runtime/workspace-runner-fat:v1.0.0",
+       "toolchains":"ghcr.io/underpass-ai/underpass-runtime/runner:gh-20260404235607"}
+  # Simplify for test: no external TLS/cert-gen dependency.
+  tls:
+    mode: disabled

charts/underpass-runtime/templates/configmap.yaml

@@ -13,6 +13,12 @@ data:
   {{- if .Values.config.notifyEscalationRoutesJSON }}
   WORKSPACE_NOTIFY_ESCALATION_ROUTES_JSON: {{ .Values.config.notifyEscalationRoutesJSON | quote }}
   {{- end }}
+  {{- if .Values.config.connProfileEndpointsJSON }}
+  WORKSPACE_CONN_PROFILE_ENDPOINTS_JSON: {{ .Values.config.connProfileEndpointsJSON | quote }}
+  {{- end }}
+  {{- if .Values.config.connProfileHostAllowlistJSON }}
+  WORKSPACE_CONN_PROFILE_HOST_ALLOWLIST_JSON: {{ .Values.config.connProfileHostAllowlistJSON | quote }}
+  {{- end }}
   WORKSPACE_ROOT: {{ .Values.workspace.root | quote }}
   ARTIFACT_ROOT: {{ .Values.artifacts.root | quote }}
   ARTIFACT_BACKEND: {{ .Values.artifacts.backend | quote }}

charts/underpass-runtime/templates/deployment.yaml

@@ -84,26 +84,19 @@ spec:
                   name: {{ .Values.artifacts.s3.existingSecret }}
                   key: {{ .Values.artifacts.s3.secretKeyKey }}
             {{- end }}
+          # tcpSocket works with or without TLS. (A grpc probe would need the
+          # standard grpc.health.v1.Health service, which this server does not
+          # implement — it exposes a custom HealthService instead.)
           livenessProbe:
-            {{- if ne $tlsMode "disabled" }}
             tcpSocket:
               port: {{ .Values.service.port }}
-            {{- else }}
-            grpc:
-              port: {{ .Values.service.port }}
-            {{- end }}
             initialDelaySeconds: 10
             periodSeconds: 30
             timeoutSeconds: 3
             failureThreshold: 3
           readinessProbe:
-            {{- if ne $tlsMode "disabled" }}
             tcpSocket:
               port: {{ .Values.service.port }}
-            {{- else }}
-            grpc:
-              port: {{ .Values.service.port }}
-            {{- end }}
             initialDelaySeconds: 5
             periodSeconds: 10
             timeoutSeconds: 3