Skip to content

Commit 1035d9d

Browse files
moulalismoulalis
committed
Merge remote-tracking branch 'upstream/main' into rhoai-3.5-ea.2
2 parents 7926d8d + a672735 commit 1035d9d

24 files changed

Lines changed: 1341 additions & 193 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ install-pre-commit-tools:
233233
$(GO) install github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.4.5
234234
@if command -v helm >/dev/null 2>&1; then \
235235
helm plugin list | grep -q unittest || \
236-
helm plugin install --verify=false --version v1.0.3 https://github.com/helm-unittest/helm-unittest.git; \
236+
helm plugin install --version v1.0.3 --verify=false https://github.com/helm-unittest/helm-unittest.git; \
237237
else \
238238
echo "helm not found, skipping helm-unittest plugin install"; \
239239
fi

‎charts/batch-gateway/dashboards/gc.json‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
{
2+
"annotations": { "list": [] },
3+
"editable": true,
4+
"fiscalYearStartMonth": 0,
5+
"graphTooltip": 1,
6+
"schemaVersion": 39,
7+
"tags": ["batch-gateway", "gc", "reconciler"],
8+
"templating": {
9+
"list": [
10+
{
11+
"current": { "selected": false, "text": "default", "value": "default" },
12+
"datasource": { "type": "prometheus" },
13+
"definition": "label_values(batch_reconciler_cycle_duration_seconds_count, namespace)",
14+
"name": "namespace",
15+
"type": "query",
16+
"refresh": 2,
17+
"includeAll": false,
18+
"multi": false
19+
}
20+
]
21+
},
22+
"time": { "from": "now-3h", "to": "now" },
23+
"title": "Batch Gateway — GC Reconciler",
24+
"uid": "batch-gateway-gc",
25+
"panels": [
26+
{
27+
"title": "Orphans Recovered (rate)",
28+
"type": "timeseries",
29+
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 8 },
30+
"targets": [
31+
{
32+
"expr": "sum(rate(batch_reconciler_orphans_recovered_total{namespace=\"$namespace\"}[5m])) by (action)",
33+
"legendFormat": "{{action}}"
34+
}
35+
],
36+
"fieldConfig": {
37+
"defaults": { "unit": "ops" }
38+
}
39+
},
40+
{
41+
"title": "Cycle Duration (p50 / p95 / p99)",
42+
"type": "timeseries",
43+
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 8 },
44+
"targets": [
45+
{
46+
"expr": "histogram_quantile(0.50, sum(rate(batch_reconciler_cycle_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (le))",
47+
"legendFormat": "p50"
48+
},
49+
{
50+
"expr": "histogram_quantile(0.95, sum(rate(batch_reconciler_cycle_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (le))",
51+
"legendFormat": "p95"
52+
},
53+
{
54+
"expr": "histogram_quantile(0.99, sum(rate(batch_reconciler_cycle_duration_seconds_bucket{namespace=\"$namespace\"}[5m])) by (le))",
55+
"legendFormat": "p99"
56+
}
57+
],
58+
"fieldConfig": {
59+
"defaults": { "unit": "s" }
60+
}
61+
},
62+
{
63+
"title": "Errors & CAS Conflicts (rate)",
64+
"type": "timeseries",
65+
"gridPos": { "x": 0, "y": 8, "w": 12, "h": 8 },
66+
"targets": [
67+
{
68+
"expr": "rate(batch_reconciler_errors_total{namespace=\"$namespace\"}[5m])",
69+
"legendFormat": "errors"
70+
},
71+
{
72+
"expr": "rate(batch_reconciler_cas_conflicts_total{namespace=\"$namespace\"}[5m])",
73+
"legendFormat": "CAS conflicts"
74+
}
75+
],
76+
"fieldConfig": {
77+
"defaults": { "unit": "ops" }
78+
}
79+
},
80+
{
81+
"title": "Stale In-Flight Cleanup (rate)",
82+
"type": "timeseries",
83+
"gridPos": { "x": 12, "y": 8, "w": 12, "h": 8 },
84+
"targets": [
85+
{
86+
"expr": "rate(batch_reconciler_stale_cleanup_total{namespace=\"$namespace\"}[5m])",
87+
"legendFormat": "stale cleanup"
88+
}
89+
],
90+
"fieldConfig": {
91+
"defaults": { "unit": "ops" }
92+
}
93+
},
94+
{
95+
"title": "Orphans Recovered (cumulative)",
96+
"type": "stat",
97+
"gridPos": { "x": 0, "y": 16, "w": 24, "h": 6 },
98+
"targets": [
99+
{
100+
"expr": "sum(batch_reconciler_orphans_recovered_total{namespace=\"$namespace\"}) by (action)",
101+
"legendFormat": "{{action}}"
102+
}
103+
]
104+
}
105+
]
106+
}

‎charts/batch-gateway/templates/gc-configmap.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ data:
99
config.yaml: |
1010
# Garbage Collector configuration
1111
dry_run: {{ .Values.gc.config.dryRun }}
12-
interval: {{ .Values.gc.config.interval | quote }}
13-
max_concurrency: {{ .Values.gc.config.maxConcurrency }}
12+
collector:
13+
interval: {{ .Values.gc.config.collector.interval | quote }}
14+
max_concurrency: {{ .Values.gc.config.collector.maxConcurrency }}
15+
metrics_addr: {{ .Values.gc.config.metricsAddr | quote }}
1416
db_client:
1517
type: {{ .Values.global.dbClient.type | quote }}
1618
redis:

‎charts/batch-gateway/templates/gc-deployment.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,29 @@ spec:
3737
args:
3838
- --config=/etc/batch-gc/config.yaml
3939
- --v={{ .Values.gc.logging.verbosity }}
40+
{{- $metricsAddr := .Values.gc.config.metricsAddr }}
41+
{{- if not (hasPrefix ":" $metricsAddr) }}
42+
{{- fail (printf "gc.config.metricsAddr must be in \":PORT\" format (e.g. \":9091\"), got %q" $metricsAddr) }}
43+
{{- end }}
44+
{{- $metricsPort := trimPrefix ":" $metricsAddr | int }}
45+
{{- if or (le $metricsPort 0) (gt $metricsPort 65535) }}
46+
{{- fail (printf "gc.config.metricsAddr port must be 1-65535, got %d" $metricsPort) }}
47+
{{- end }}
48+
ports:
49+
- name: metrics
50+
containerPort: {{ $metricsPort }}
51+
protocol: TCP
52+
startupProbe:
53+
httpGet:
54+
path: /health
55+
port: metrics
56+
periodSeconds: 5
57+
failureThreshold: 60
58+
livenessProbe:
59+
httpGet:
60+
path: /health
61+
port: metrics
62+
periodSeconds: 30
4063
resources:
4164
{{- toYaml .Values.gc.resources | nindent 12 }}
4265
volumeMounts:

‎charts/batch-gateway/templates/gc-podmonitor.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{{- if and .Values.gc.enabled .Values.gc.podMonitor.enabled }}
2+
apiVersion: monitoring.coreos.com/v1
3+
kind: PodMonitor
4+
metadata:
5+
name: {{ include "batch-gateway.gc.fullname" . }}
6+
labels:
7+
{{- include "batch-gateway.gc.labels" . | nindent 4 }}
8+
{{- with .Values.gc.podMonitor.labels }}
9+
{{- toYaml . | nindent 4 }}
10+
{{- end }}
11+
spec:
12+
namespaceSelector:
13+
matchNames:
14+
- {{ .Release.Namespace }}
15+
selector:
16+
matchLabels:
17+
{{- include "batch-gateway.gc.selectorLabels" . | nindent 6 }}
18+
podMetricsEndpoints:
19+
- port: metrics
20+
path: /metrics
21+
interval: {{ .Values.gc.podMonitor.interval | quote }}
22+
{{- end }}

‎charts/batch-gateway/templates/prometheus-rule.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,4 +68,16 @@ spec:
6868
summary: Batch Gateway workers are saturated
6969
description: Worker utilization has been above {{ .Values.prometheusRule.rules.workersSaturated.thresholdRatio }} for {{ .Values.prometheusRule.rules.workersSaturated.for }}.
7070
{{- end }}
71+
{{- if .Values.prometheusRule.rules.reconcilerErrors.enabled }}
72+
- alert: BatchReconcilerErrors
73+
expr: |
74+
increase(batch_reconciler_errors_total{namespace="{{ .Release.Namespace }}"}[{{ .Values.prometheusRule.rules.reconcilerErrors.window }}])
75+
> 0
76+
for: {{ .Values.prometheusRule.rules.reconcilerErrors.for | quote }}
77+
labels:
78+
severity: warning
79+
annotations:
80+
summary: Batch reconciler is encountering errors
81+
description: The GC orphan reconciler encountered errors in the last {{ .Values.prometheusRule.rules.reconcilerErrors.window }}.
82+
{{- end }}
7183
{{- end }}

‎charts/batch-gateway/tests/gc-configmap_test.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,20 @@ tests:
1111
pattern: 'dry_run: false'
1212
- matchRegex:
1313
path: data["config.yaml"]
14-
pattern: 'interval: "30m"'
14+
pattern: 'collector:\n\s+interval: "30m"'
1515
- matchRegex:
1616
path: data["config.yaml"]
1717
pattern: 'db_client:\n\s+type: "postgresql"'
1818
- matchRegex:
1919
path: data["config.yaml"]
2020
pattern: 'max_concurrency: 10'
21+
- matchRegex:
22+
path: data["config.yaml"]
23+
pattern: 'metrics_addr: ":9091"'
2124

2225
- it: should render custom maxConcurrency
2326
set:
24-
gc.config.maxConcurrency: 20
27+
gc.config.collector.maxConcurrency: 20
2528
asserts:
2629
- matchRegex:
2730
path: data["config.yaml"]
@@ -37,7 +40,7 @@ tests:
3740

3841
- it: should render custom interval
3942
set:
40-
gc.config.interval: "1h"
43+
gc.config.collector.interval: "1h"
4144
asserts:
4245
- matchRegex:
4346
path: data["config.yaml"]

‎charts/batch-gateway/tests/observability_test.yaml‎

Lines changed: 97 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ templates:
44
- templates/apiserver-configmap.yaml
55
- templates/processor-podmonitor.yaml
66
- templates/processor-configmap.yaml
7+
- templates/gc-podmonitor.yaml
78
- templates/prometheus-rule.yaml
89
- templates/grafana-dashboards-configmap.yaml
910
# Required: fs file client (the default) needs a pvcName to pass template validation.
@@ -34,6 +35,15 @@ tests:
3435
value: "30s"
3536
template: templates/apiserver-servicemonitor.yaml
3637

38+
- it: should include RHOAI monitoring label on ServiceMonitor by default
39+
set:
40+
apiserver.serviceMonitor.enabled: true
41+
asserts:
42+
- equal:
43+
path: metadata.labels["monitoring.opendatahub.io/scrape"]
44+
value: "true"
45+
template: templates/apiserver-servicemonitor.yaml
46+
3747
- it: should apply custom ServiceMonitor interval
3848
set:
3949
apiserver.serviceMonitor.enabled: true
@@ -77,6 +87,15 @@ tests:
7787
value: "30s"
7888
template: templates/processor-podmonitor.yaml
7989

90+
- it: should include RHOAI monitoring label on PodMonitor by default
91+
set:
92+
processor.podMonitor.enabled: true
93+
asserts:
94+
- equal:
95+
path: metadata.labels["monitoring.opendatahub.io/scrape"]
96+
value: "true"
97+
template: templates/processor-podmonitor.yaml
98+
8099
- it: should not render PodMonitor when processor is disabled
81100
set:
82101
processor.enabled: false
@@ -86,6 +105,39 @@ tests:
86105
count: 0
87106
template: templates/processor-podmonitor.yaml
88107

108+
# --- GC PodMonitor ---
109+
110+
- it: should not render GC PodMonitor when disabled (default)
111+
asserts:
112+
- hasDocuments:
113+
count: 0
114+
template: templates/gc-podmonitor.yaml
115+
116+
- it: should render GC PodMonitor when enabled
117+
set:
118+
gc.podMonitor.enabled: true
119+
asserts:
120+
- isKind:
121+
of: PodMonitor
122+
template: templates/gc-podmonitor.yaml
123+
- equal:
124+
path: spec.podMetricsEndpoints[0].port
125+
value: metrics
126+
template: templates/gc-podmonitor.yaml
127+
- equal:
128+
path: spec.podMetricsEndpoints[0].interval
129+
value: "30s"
130+
template: templates/gc-podmonitor.yaml
131+
132+
- it: should not render GC PodMonitor when gc is disabled
133+
set:
134+
gc.enabled: false
135+
gc.podMonitor.enabled: true
136+
asserts:
137+
- hasDocuments:
138+
count: 0
139+
template: templates/gc-podmonitor.yaml
140+
89141
# --- PrometheusRule ---
90142

91143
- it: should not render PrometheusRule when disabled (default)
@@ -106,7 +158,7 @@ tests:
106158
value: batch-gateway.rules
107159
template: templates/prometheus-rule.yaml
108160

109-
- it: should render all four alert rules when enabled
161+
- it: should render all five alert rules when enabled
110162
set:
111163
prometheusRule.enabled: true
112164
asserts:
@@ -134,6 +186,12 @@ tests:
134186
alert: BatchGatewayWorkersSaturated
135187
any: true
136188
template: templates/prometheus-rule.yaml
189+
- contains:
190+
path: spec.groups[0].rules
191+
content:
192+
alert: BatchReconcilerErrors
193+
any: true
194+
template: templates/prometheus-rule.yaml
137195

138196
# --- PrometheusRule expr metric name guards ---
139197

@@ -177,6 +235,15 @@ tests:
177235
pattern: total_workers
178236
template: templates/prometheus-rule.yaml
179237

238+
- it: should use batch_reconciler_errors_total in ReconcilerErrors expr
239+
set:
240+
prometheusRule.enabled: true
241+
asserts:
242+
- matchRegex:
243+
path: spec.groups[0].rules[4].expr
244+
pattern: batch_reconciler_errors_total
245+
template: templates/prometheus-rule.yaml
246+
180247
- it: should not reference tenantID in any PrometheusRule expr
181248
set:
182249
prometheusRule.enabled: true
@@ -197,6 +264,10 @@ tests:
197264
path: spec.groups[0].rules[3].expr
198265
pattern: tenantID
199266
template: templates/prometheus-rule.yaml
267+
- notMatchRegex:
268+
path: spec.groups[0].rules[4].expr
269+
pattern: tenantID
270+
template: templates/prometheus-rule.yaml
200271

201272
# --- Grafana Dashboards ConfigMap ---
202273

@@ -258,6 +329,31 @@ tests:
258329
pattern: tenantID
259330
template: templates/grafana-dashboards-configmap.yaml
260331

332+
- it: should reference expected GC reconciler metric names in dashboard JSON
333+
set:
334+
grafana.dashboards.enabled: true
335+
asserts:
336+
- matchRegex:
337+
path: data["gc.json"]
338+
pattern: batch_reconciler_orphans_recovered_total
339+
template: templates/grafana-dashboards-configmap.yaml
340+
- matchRegex:
341+
path: data["gc.json"]
342+
pattern: batch_reconciler_cycle_duration_seconds_bucket
343+
template: templates/grafana-dashboards-configmap.yaml
344+
- matchRegex:
345+
path: data["gc.json"]
346+
pattern: batch_reconciler_errors_total
347+
template: templates/grafana-dashboards-configmap.yaml
348+
- matchRegex:
349+
path: data["gc.json"]
350+
pattern: batch_reconciler_cas_conflicts_total
351+
template: templates/grafana-dashboards-configmap.yaml
352+
- matchRegex:
353+
path: data["gc.json"]
354+
pattern: batch_reconciler_stale_cleanup_total
355+
template: templates/grafana-dashboards-configmap.yaml
356+
261357
- it: should reference expected apiserver metric names in dashboard JSON
262358
set:
263359
grafana.dashboards.enabled: true

0 commit comments

Comments
 (0)