fix: Notify k8s of broken pool executors

charts/datafold-manager/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: datafold-manager
 description: Helm chart for Datafold Operator
 type: application
-version: 0.1.100
+version: 0.1.101
 appVersion: "1.0.0"
 icon: https://www.datafold.com/logo.png
 

charts/datafold-manager/values.yaml

@@ -18,7 +18,7 @@ operator:
   # Operator image configuration
   image:
     repository: us-docker.pkg.dev/datadiff-mm/datafold/datafold-operator
-    tag: "1.1.70"
+    tag: "1.1.71"
     pullPolicy: Always
 
   # Operator deployment configuration

charts/datafold/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: datafold
 description: Helm chart package to deploy Datafold on kubernetes.
 type: application
-version: 0.10.84
+version: 0.10.85
 appVersion: "1.0.0"
 icon: https://www.datafold.com/logo.png
 

charts/datafold/charts/worker-temporal/templates/deployment.yaml

@@ -103,14 +103,35 @@ spec:
             - name: TEMPORAL_METRICS_BIND_ADDRESS
               value: "0.0.0.0:{{ .Values.metrics.port }}"
             {{- end }}
+            {{- if .Values.health.enabled }}
+            - name: TEMPORAL_HEALTH_PORT
+              value: {{ .Values.health.port | quote }}
+            {{- end }}
             {{- with .Values.extraEnv }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
-          {{- if .Values.metrics.enabled }}
+          {{- if or .Values.metrics.enabled .Values.health.enabled }}
           ports:
+            {{- if .Values.metrics.enabled }}
             - name: metrics
               containerPort: {{ .Values.metrics.port }}
               protocol: TCP
+            {{- end }}
+            {{- if .Values.health.enabled }}
+            - name: health
+              containerPort: {{ .Values.health.port }}
+              protocol: TCP
+            {{- end }}
+          {{- end }}
+          {{- if .Values.health.enabled }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: {{ .Values.health.port }}
+            initialDelaySeconds: {{ .Values.health.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.health.livenessProbe.periodSeconds }}
+            failureThreshold: {{ .Values.health.livenessProbe.failureThreshold }}
+            timeoutSeconds: {{ .Values.health.livenessProbe.timeoutSeconds }}
           {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}

charts/datafold/charts/worker-temporal/values.yaml

@@ -76,6 +76,20 @@ metrics:
   enabled: true
   port: 9090
 
+health:
+  # Enables the HTTP liveness probe endpoint (TEMPORAL_HEALTH_PORT).
+  # When the activity executor becomes broken the endpoint returns 503, which
+  # triggers K8s to send SIGTERM and schedule a replacement pod immediately.
+  # The old pod continues draining within terminationGracePeriodSeconds so
+  # in-flight work on healthy threads can still complete.
+  enabled: true
+  port: 8091
+  livenessProbe:
+    initialDelaySeconds: 30
+    periodSeconds: 15
+    failureThreshold: 4
+    timeoutSeconds: 5
+
 extraEnv: []
 
 volumes: []