Skip to content

Commit 1e4b3cc

Browse files
tgarciaitgarciai
authored
Merge pull request #157 from underpass-ai/fix/tool-learning-export-lake
fix(tool-learning): deploy export-lake producer so the telemetry lake is populated
2 parents a01b8bd + 7a869d8 commit 1e4b3cc

9 files changed

Lines changed: 339 additions & 141 deletions

File tree

‎charts/underpass-runtime/templates/_tool-learning.tpl‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
{{/*
2+
Shared pod plumbing for the tool-learning CronJobs.
3+
4+
Both the policy-computation job (cronjob-tool-learning.yaml) and the export-lake
5+
producer (cronjob-tool-learning-export.yaml) talk to the same Valkey, S3/MinIO
6+
lake and TLS material, so the environment and TLS volumes are defined once here
7+
to keep producer and consumer in sync. Call with the root context, e.g.
8+
env:
9+
{{- include "underpass-runtime.toolLearning.env" $ | nindent 16 }}
10+
*/}}
11+
{{- define "underpass-runtime.toolLearning.env" -}}
12+
- name: HOME
13+
value: /tmp
14+
- name: LOG_LEVEL
15+
value: {{ .Values.toolLearning.logLevel | quote }}
16+
{{- /* S3 / MinIO */}}
17+
- name: S3_ENDPOINT
18+
value: {{ .Values.toolLearning.s3.endpoint | quote }}
19+
- name: S3_REGION
20+
value: {{ .Values.toolLearning.s3.region | quote }}
21+
- name: S3_USE_SSL
22+
value: {{ or .Values.toolLearning.s3.useSSL (default false .Values.s3Tls.enabled) | quote }}
23+
- name: LAKE_BUCKET
24+
value: {{ .Values.toolLearning.s3.lakeBucket | quote }}
25+
- name: AUDIT_BUCKET
26+
value: {{ .Values.toolLearning.s3.auditBucket | quote }}
27+
{{- if .Values.toolLearning.s3.existingSecret }}
28+
- name: S3_ACCESS_KEY
29+
valueFrom:
30+
secretKeyRef:
31+
name: {{ .Values.toolLearning.s3.existingSecret }}
32+
key: {{ .Values.toolLearning.s3.accessKeyKey }}
33+
- name: S3_SECRET_KEY
34+
valueFrom:
35+
secretKeyRef:
36+
name: {{ .Values.toolLearning.s3.existingSecret }}
37+
key: {{ .Values.toolLearning.s3.secretKeyKey }}
38+
{{- end }}
39+
{{- /* S3 TLS */}}
40+
{{- if default false .Values.s3Tls.enabled }}
41+
{{- if and .Values.s3Tls.existingSecret .Values.s3Tls.keys.ca }}
42+
- name: S3_CA_PATH
43+
value: {{ printf "%s/%s" .Values.s3Tls.mountPath .Values.s3Tls.keys.ca | quote }}
44+
{{- end }}
45+
{{- if and .Values.s3Tls.keys.cert .Values.s3Tls.keys.key }}
46+
- name: S3_CERT_PATH
47+
value: {{ printf "%s/%s" .Values.s3Tls.mountPath .Values.s3Tls.keys.cert | quote }}
48+
- name: S3_KEY_PATH
49+
value: {{ printf "%s/%s" .Values.s3Tls.mountPath .Values.s3Tls.keys.key | quote }}
50+
{{- end }}
51+
{{- end }}
52+
{{- /* Valkey */}}
53+
- name: VALKEY_ADDR
54+
value: "{{ .Values.toolLearning.valkey.host }}:{{ .Values.toolLearning.valkey.port }}"
55+
- name: VALKEY_DB
56+
value: {{ .Values.toolLearning.valkey.db | quote }}
57+
- name: VALKEY_KEY_PREFIX
58+
value: {{ .Values.toolLearning.valkey.keyPrefix | quote }}
59+
- name: VALKEY_TTL
60+
value: {{ .Values.toolLearning.valkey.ttl | quote }}
61+
- name: VALKEY_TELEMETRY_PREFIX
62+
value: {{ .Values.toolLearning.valkey.telemetryPrefix | quote }}
63+
{{- if .Values.toolLearning.valkey.existingSecret }}
64+
- name: VALKEY_PASSWORD
65+
valueFrom:
66+
secretKeyRef:
67+
name: {{ .Values.toolLearning.valkey.existingSecret }}
68+
key: {{ .Values.toolLearning.valkey.passwordKey | default "password" }}
69+
{{- end }}
70+
{{- /* NATS */}}
71+
- name: NATS_URL
72+
value: {{ .Values.toolLearning.nats.url | quote }}
73+
{{- /* NATS TLS */}}
74+
{{- if ne (default "disabled" .Values.natsTls.mode) "disabled" }}
75+
- name: NATS_TLS_MODE
76+
value: {{ .Values.natsTls.mode | quote }}
77+
{{- if and .Values.natsTls.existingSecret .Values.natsTls.keys.ca }}
78+
- name: NATS_TLS_CA_PATH
79+
value: {{ printf "%s/%s" .Values.natsTls.mountPath .Values.natsTls.keys.ca | quote }}
80+
{{- end }}
81+
{{- if and .Values.natsTls.keys.cert .Values.natsTls.keys.key }}
82+
- name: NATS_TLS_CERT_PATH
83+
value: {{ printf "%s/%s" .Values.natsTls.mountPath .Values.natsTls.keys.cert | quote }}
84+
- name: NATS_TLS_KEY_PATH
85+
value: {{ printf "%s/%s" .Values.natsTls.mountPath .Values.natsTls.keys.key | quote }}
86+
{{- end }}
87+
{{- end }}
88+
{{- /* Valkey TLS */}}
89+
{{- if .Values.valkeyTls.enabled }}
90+
- name: VALKEY_TLS_ENABLED
91+
value: "true"
92+
{{- if and .Values.valkeyTls.existingSecret .Values.valkeyTls.keys.ca }}
93+
- name: VALKEY_TLS_CA_PATH
94+
value: {{ printf "%s/%s" .Values.valkeyTls.mountPath .Values.valkeyTls.keys.ca | quote }}
95+
{{- end }}
96+
{{- if and .Values.valkeyTls.keys.cert .Values.valkeyTls.keys.key }}
97+
- name: VALKEY_TLS_CERT_PATH
98+
value: {{ printf "%s/%s" .Values.valkeyTls.mountPath .Values.valkeyTls.keys.cert | quote }}
99+
- name: VALKEY_TLS_KEY_PATH
100+
value: {{ printf "%s/%s" .Values.valkeyTls.mountPath .Values.valkeyTls.keys.key | quote }}
101+
{{- end }}
102+
{{- end }}
103+
{{- /* DuckDB/libcurl custom CA + client cert for S3 mTLS */}}
104+
{{- if and (default false .Values.s3Tls.enabled) .Values.s3Tls.existingSecret .Values.s3Tls.keys.ca }}
105+
- name: SSL_CERT_FILE
106+
value: {{ printf "%s/%s" .Values.s3Tls.mountPath .Values.s3Tls.keys.ca | quote }}
107+
{{- if and .Values.s3Tls.keys.cert .Values.s3Tls.keys.key }}
108+
- name: CURL_SSLCERT
109+
value: {{ printf "%s/%s" .Values.s3Tls.mountPath .Values.s3Tls.keys.cert | quote }}
110+
- name: CURL_SSLKEY
111+
value: {{ printf "%s/%s" .Values.s3Tls.mountPath .Values.s3Tls.keys.key | quote }}
112+
{{- end }}
113+
{{- else if and .Values.valkeyTls.enabled .Values.valkeyTls.existingSecret .Values.valkeyTls.keys.ca }}
114+
{{- /* Fallback: reuse Valkey CA for DuckDB/libcurl when s3Tls is not configured */}}
115+
- name: SSL_CERT_FILE
116+
value: {{ printf "%s/%s" .Values.valkeyTls.mountPath .Values.valkeyTls.keys.ca | quote }}
117+
{{- end }}
118+
{{- end -}}
119+
120+
{{- define "underpass-runtime.toolLearning.volumeMounts" -}}
121+
- name: tmp
122+
mountPath: /tmp
123+
{{- if and .Values.valkeyTls.enabled .Values.valkeyTls.existingSecret }}
124+
- name: valkey-tls
125+
mountPath: {{ .Values.valkeyTls.mountPath }}
126+
readOnly: true
127+
{{- end }}
128+
{{- if and (ne (default "disabled" .Values.natsTls.mode) "disabled") .Values.natsTls.existingSecret }}
129+
- name: nats-tls
130+
mountPath: {{ .Values.natsTls.mountPath }}
131+
readOnly: true
132+
{{- end }}
133+
{{- if and (default false .Values.s3Tls.enabled) .Values.s3Tls.existingSecret }}
134+
- name: s3-tls
135+
mountPath: {{ .Values.s3Tls.mountPath }}
136+
readOnly: true
137+
{{- end }}
138+
{{- end -}}
139+
140+
{{- define "underpass-runtime.toolLearning.volumes" -}}
141+
- name: tmp
142+
emptyDir:
143+
sizeLimit: 256Mi
144+
{{- if and .Values.valkeyTls.enabled .Values.valkeyTls.existingSecret }}
145+
- name: valkey-tls
146+
secret:
147+
secretName: {{ .Values.valkeyTls.existingSecret }}
148+
{{- end }}
149+
{{- if and (ne (default "disabled" .Values.natsTls.mode) "disabled") .Values.natsTls.existingSecret }}
150+
- name: nats-tls
151+
secret:
152+
secretName: {{ .Values.natsTls.existingSecret }}
153+
{{- end }}
154+
{{- if and (default false .Values.s3Tls.enabled) .Values.s3Tls.existingSecret }}
155+
- name: s3-tls
156+
secret:
157+
secretName: {{ .Values.s3Tls.existingSecret }}
158+
{{- end }}
159+
{{- end -}}

‎charts/underpass-runtime/templates/cronjob-tool-learning-export.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{{- if and .Values.toolLearning.enabled .Values.toolLearning.exportLake.enabled }}
2+
{{- $fullName := include "underpass-runtime.fullname" . -}}
3+
{{- /*
4+
export-lake is the PRODUCER half of the tool-learning pipeline: it reads
5+
telemetry records from Valkey and writes them as Hive-partitioned Parquet to the
6+
S3 telemetry lake. The compute CronJobs (cronjob-tool-learning.yaml) read that
7+
lake, so this job MUST run before them — schedule it ahead of the compute crons.
8+
*/}}
9+
apiVersion: batch/v1
10+
kind: CronJob
11+
metadata:
12+
name: {{ $fullName }}-tool-learning-export
13+
labels:
14+
{{- include "underpass-runtime.labels" . | nindent 4 }}
15+
app.kubernetes.io/component: tool-learning-export
16+
spec:
17+
schedule: {{ .Values.toolLearning.exportLake.schedule | quote }}
18+
concurrencyPolicy: Forbid
19+
successfulJobsHistoryLimit: {{ .Values.toolLearning.successfulJobsHistoryLimit }}
20+
failedJobsHistoryLimit: {{ .Values.toolLearning.failedJobsHistoryLimit }}
21+
startingDeadlineSeconds: {{ .Values.toolLearning.startingDeadlineSeconds }}
22+
jobTemplate:
23+
metadata:
24+
labels:
25+
{{- include "underpass-runtime.selectorLabels" . | nindent 8 }}
26+
app.kubernetes.io/component: tool-learning-export
27+
spec:
28+
activeDeadlineSeconds: {{ .Values.toolLearning.activeDeadlineSeconds }}
29+
backoffLimit: {{ .Values.toolLearning.backoffLimit }}
30+
template:
31+
metadata:
32+
labels:
33+
{{- include "underpass-runtime.selectorLabels" . | nindent 12 }}
34+
app.kubernetes.io/component: tool-learning-export
35+
{{- with .Values.toolLearning.podAnnotations }}
36+
annotations:
37+
{{- toYaml . | nindent 12 }}
38+
{{- end }}
39+
spec:
40+
{{- with .Values.imagePullSecrets }}
41+
imagePullSecrets:
42+
{{- toYaml . | nindent 12 }}
43+
{{- end }}
44+
serviceAccountName: {{ include "underpass-runtime.serviceAccountName" . }}
45+
automountServiceAccountToken: false
46+
restartPolicy: OnFailure
47+
securityContext:
48+
{{- toYaml .Values.podSecurityContext | nindent 12 }}
49+
containers:
50+
- name: export-lake
51+
securityContext:
52+
allowPrivilegeEscalation: false
53+
readOnlyRootFilesystem: true
54+
capabilities:
55+
drop:
56+
- ALL
57+
image: "{{ .Values.toolLearning.image.repository }}:{{ .Values.toolLearning.image.tag | default .Chart.AppVersion }}"
58+
imagePullPolicy: {{ .Values.toolLearning.image.pullPolicy }}
59+
command:
60+
- /app/export-lake
61+
env:
62+
{{- include "underpass-runtime.toolLearning.env" . | nindent 16 }}
63+
resources:
64+
{{- toYaml .Values.toolLearning.resources | nindent 16 }}
65+
volumeMounts:
66+
{{- include "underpass-runtime.toolLearning.volumeMounts" . | nindent 16 }}
67+
volumes:
68+
{{- include "underpass-runtime.toolLearning.volumes" . | nindent 12 }}
69+
{{- with .Values.toolLearning.nodeSelector }}
70+
nodeSelector:
71+
{{- toYaml . | nindent 12 }}
72+
{{- end }}
73+
{{- with .Values.toolLearning.tolerations }}
74+
tolerations:
75+
{{- toYaml . | nindent 12 }}
76+
{{- end }}
77+
{{- with .Values.toolLearning.affinity }}
78+
affinity:
79+
{{- toYaml . | nindent 12 }}
80+
{{- end }}
81+
{{- end }}

‎charts/underpass-runtime/templates/cronjob-tool-learning.yaml‎

Lines changed: 3 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -68,149 +68,13 @@ spec:
6868
{{- end }}
6969
{{- end }}
7070
env:
71-
- name: HOME
72-
value: /tmp
73-
- name: LOG_LEVEL
74-
value: {{ $.Values.toolLearning.logLevel | quote }}
75-
{{- /* S3 / MinIO */}}
76-
- name: S3_ENDPOINT
77-
value: {{ $.Values.toolLearning.s3.endpoint | quote }}
78-
- name: S3_REGION
79-
value: {{ $.Values.toolLearning.s3.region | quote }}
80-
- name: S3_USE_SSL
81-
value: {{ or $.Values.toolLearning.s3.useSSL (default false $.Values.s3Tls.enabled) | quote }}
82-
- name: LAKE_BUCKET
83-
value: {{ $.Values.toolLearning.s3.lakeBucket | quote }}
84-
- name: AUDIT_BUCKET
85-
value: {{ $.Values.toolLearning.s3.auditBucket | quote }}
86-
{{- if $.Values.toolLearning.s3.existingSecret }}
87-
- name: S3_ACCESS_KEY
88-
valueFrom:
89-
secretKeyRef:
90-
name: {{ $.Values.toolLearning.s3.existingSecret }}
91-
key: {{ $.Values.toolLearning.s3.accessKeyKey }}
92-
- name: S3_SECRET_KEY
93-
valueFrom:
94-
secretKeyRef:
95-
name: {{ $.Values.toolLearning.s3.existingSecret }}
96-
key: {{ $.Values.toolLearning.s3.secretKeyKey }}
97-
{{- end }}
98-
{{- /* S3 TLS */}}
99-
{{- if default false $.Values.s3Tls.enabled }}
100-
{{- if and $.Values.s3Tls.existingSecret $.Values.s3Tls.keys.ca }}
101-
- name: S3_CA_PATH
102-
value: {{ printf "%s/%s" $.Values.s3Tls.mountPath $.Values.s3Tls.keys.ca | quote }}
103-
{{- end }}
104-
{{- if and $.Values.s3Tls.keys.cert $.Values.s3Tls.keys.key }}
105-
- name: S3_CERT_PATH
106-
value: {{ printf "%s/%s" $.Values.s3Tls.mountPath $.Values.s3Tls.keys.cert | quote }}
107-
- name: S3_KEY_PATH
108-
value: {{ printf "%s/%s" $.Values.s3Tls.mountPath $.Values.s3Tls.keys.key | quote }}
109-
{{- end }}
110-
{{- end }}
111-
{{- /* Valkey */}}
112-
- name: VALKEY_ADDR
113-
value: "{{ $.Values.toolLearning.valkey.host }}:{{ $.Values.toolLearning.valkey.port }}"
114-
- name: VALKEY_DB
115-
value: {{ $.Values.toolLearning.valkey.db | quote }}
116-
- name: VALKEY_KEY_PREFIX
117-
value: {{ $.Values.toolLearning.valkey.keyPrefix | quote }}
118-
- name: VALKEY_TTL
119-
value: {{ $.Values.toolLearning.valkey.ttl | quote }}
120-
{{- if $.Values.toolLearning.valkey.existingSecret }}
121-
- name: VALKEY_PASSWORD
122-
valueFrom:
123-
secretKeyRef:
124-
name: {{ $.Values.toolLearning.valkey.existingSecret }}
125-
key: {{ $.Values.toolLearning.valkey.passwordKey | default "password" }}
126-
{{- end }}
127-
{{- /* NATS */}}
128-
- name: NATS_URL
129-
value: {{ $.Values.toolLearning.nats.url | quote }}
130-
{{- /* NATS TLS */}}
131-
{{- if ne (default "disabled" $.Values.natsTls.mode) "disabled" }}
132-
- name: NATS_TLS_MODE
133-
value: {{ $.Values.natsTls.mode | quote }}
134-
{{- if and $.Values.natsTls.existingSecret $.Values.natsTls.keys.ca }}
135-
- name: NATS_TLS_CA_PATH
136-
value: {{ printf "%s/%s" $.Values.natsTls.mountPath $.Values.natsTls.keys.ca | quote }}
137-
{{- end }}
138-
{{- if and $.Values.natsTls.keys.cert $.Values.natsTls.keys.key }}
139-
- name: NATS_TLS_CERT_PATH
140-
value: {{ printf "%s/%s" $.Values.natsTls.mountPath $.Values.natsTls.keys.cert | quote }}
141-
- name: NATS_TLS_KEY_PATH
142-
value: {{ printf "%s/%s" $.Values.natsTls.mountPath $.Values.natsTls.keys.key | quote }}
143-
{{- end }}
144-
{{- end }}
145-
{{- /* Valkey TLS */}}
146-
{{- if $.Values.valkeyTls.enabled }}
147-
- name: VALKEY_TLS_ENABLED
148-
value: "true"
149-
{{- if and $.Values.valkeyTls.existingSecret $.Values.valkeyTls.keys.ca }}
150-
- name: VALKEY_TLS_CA_PATH
151-
value: {{ printf "%s/%s" $.Values.valkeyTls.mountPath $.Values.valkeyTls.keys.ca | quote }}
152-
{{- end }}
153-
{{- if and $.Values.valkeyTls.keys.cert $.Values.valkeyTls.keys.key }}
154-
- name: VALKEY_TLS_CERT_PATH
155-
value: {{ printf "%s/%s" $.Values.valkeyTls.mountPath $.Values.valkeyTls.keys.cert | quote }}
156-
- name: VALKEY_TLS_KEY_PATH
157-
value: {{ printf "%s/%s" $.Values.valkeyTls.mountPath $.Values.valkeyTls.keys.key | quote }}
158-
{{- end }}
159-
{{- end }}
160-
{{- /* DuckDB/libcurl custom CA + client cert for S3 mTLS */}}
161-
{{- if and (default false $.Values.s3Tls.enabled) $.Values.s3Tls.existingSecret $.Values.s3Tls.keys.ca }}
162-
- name: SSL_CERT_FILE
163-
value: {{ printf "%s/%s" $.Values.s3Tls.mountPath $.Values.s3Tls.keys.ca | quote }}
164-
{{- if and $.Values.s3Tls.keys.cert $.Values.s3Tls.keys.key }}
165-
- name: CURL_SSLCERT
166-
value: {{ printf "%s/%s" $.Values.s3Tls.mountPath $.Values.s3Tls.keys.cert | quote }}
167-
- name: CURL_SSLKEY
168-
value: {{ printf "%s/%s" $.Values.s3Tls.mountPath $.Values.s3Tls.keys.key | quote }}
169-
{{- end }}
170-
{{- else if and $.Values.valkeyTls.enabled $.Values.valkeyTls.existingSecret $.Values.valkeyTls.keys.ca }}
171-
{{- /* Fallback: reuse Valkey CA for DuckDB/libcurl when s3Tls is not configured */}}
172-
- name: SSL_CERT_FILE
173-
value: {{ printf "%s/%s" $.Values.valkeyTls.mountPath $.Values.valkeyTls.keys.ca | quote }}
174-
{{- end }}
71+
{{- include "underpass-runtime.toolLearning.env" $ | nindent 16 }}
17572
resources:
17673
{{- toYaml $.Values.toolLearning.resources | nindent 16 }}
17774
volumeMounts:
178-
- name: tmp
179-
mountPath: /tmp
180-
{{- if and $.Values.valkeyTls.enabled $.Values.valkeyTls.existingSecret }}
181-
- name: valkey-tls
182-
mountPath: {{ $.Values.valkeyTls.mountPath }}
183-
readOnly: true
184-
{{- end }}
185-
{{- if and (ne (default "disabled" $.Values.natsTls.mode) "disabled") $.Values.natsTls.existingSecret }}
186-
- name: nats-tls
187-
mountPath: {{ $.Values.natsTls.mountPath }}
188-
readOnly: true
189-
{{- end }}
190-
{{- if and (default false $.Values.s3Tls.enabled) $.Values.s3Tls.existingSecret }}
191-
- name: s3-tls
192-
mountPath: {{ $.Values.s3Tls.mountPath }}
193-
readOnly: true
194-
{{- end }}
75+
{{- include "underpass-runtime.toolLearning.volumeMounts" $ | nindent 16 }}
19576
volumes:
196-
- name: tmp
197-
emptyDir:
198-
sizeLimit: 256Mi
199-
{{- if and $.Values.valkeyTls.enabled $.Values.valkeyTls.existingSecret }}
200-
- name: valkey-tls
201-
secret:
202-
secretName: {{ $.Values.valkeyTls.existingSecret }}
203-
{{- end }}
204-
{{- if and (ne (default "disabled" $.Values.natsTls.mode) "disabled") $.Values.natsTls.existingSecret }}
205-
- name: nats-tls
206-
secret:
207-
secretName: {{ $.Values.natsTls.existingSecret }}
208-
{{- end }}
209-
{{- if and (default false $.Values.s3Tls.enabled) $.Values.s3Tls.existingSecret }}
210-
- name: s3-tls
211-
secret:
212-
secretName: {{ $.Values.s3Tls.existingSecret }}
213-
{{- end }}
77+
{{- include "underpass-runtime.toolLearning.volumes" $ | nindent 12 }}
21478
{{- with $.Values.toolLearning.nodeSelector }}
21579
nodeSelector:
21680
{{- toYaml . | nindent 12 }}

0 commit comments

Comments
 (0)