Skip to content

Commit 11de4c5

Browse files
sbernauersbernauer
committed
WIP: Scrape everything
1 parent 49f1756 commit 11de4c5

1 file changed

Lines changed: 206 additions & 9 deletions

File tree

‎stacks/_templates/prometheus-service-monitor.yaml‎

Lines changed: 206 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,236 @@
1+
# ### Products
2+
#
3+
# Use something like this to check for metrics:
4+
# count by(job) ({__name__!=""})
5+
#
6+
# See https://github.com/stackabletech/issues/issues/735
7+
#
8+
# - [x] Airflow - exporter
9+
# - [x] Druid - native
10+
# - [x] HBase - native
11+
# - [x] Hadoop HDFS - native
12+
# - [x] Hive - exporter
13+
# - [x] Kafka - exporter
14+
# - [x] NiFi 1 - native
15+
# - [ ] NiFi 2 - native - partially working, needs mTLS
16+
# - [ ] OpenSearch
17+
# - [ ] Spark - native
18+
# - [x] Superset - exporter
19+
# - [x] Trino - native
20+
# - [x] ZooKeeper - native
21+
# - [x] OPA - native
122
---
223
apiVersion: monitoring.coreos.com/v1
324
kind: ServiceMonitor
425
metadata:
5-
name: scrape-label
26+
name: stackable
627
labels:
728
stackable.tech/vendor: Stackable
829
release: prometheus
930
spec:
31+
namespaceSelector:
32+
any: true
33+
selector:
34+
matchLabels:
35+
stackable.tech/vendor: Stackable
36+
prometheus.io/scrape: "true"
37+
matchExpressions:
38+
- key: app.kubernetes.io/name
39+
operator: In
40+
values:
41+
- airflow
42+
- druid
43+
- hive
44+
- kafka
45+
- nifi # This only works for NiFi 1, NiFi 2 has a special ServiceMonitor below
46+
- opa
47+
- superset
48+
- trino
1049
endpoints:
11-
- port: metrics
50+
- scheme: http
51+
port: metrics
52+
path: /metrics
53+
jobLabel: app.kubernetes.io/instance
54+
---
55+
apiVersion: monitoring.coreos.com/v1
56+
kind: ServiceMonitor
57+
metadata:
58+
name: stackable-native-metrics
59+
labels:
60+
stackable.tech/vendor: Stackable
61+
release: prometheus
62+
spec:
63+
namespaceSelector:
64+
any: true
1265
selector:
1366
matchLabels:
67+
stackable.tech/vendor: Stackable
1468
prometheus.io/scrape: "true"
69+
matchExpressions:
70+
- key: app.kubernetes.io/name
71+
operator: In
72+
values:
73+
- zookeeper
74+
endpoints:
75+
- scheme: http
76+
port: native-metrics
77+
path: /metrics
78+
jobLabel: app.kubernetes.io/instance
79+
---
80+
# Kafka is special in that the operator totally messes up services:
81+
# 1. The metrics Service is missing
82+
# 2. The role level simple-kafka-broker-default has the prometheus.io/scrape label, but exposes no ports...
83+
# 3. The role level simple-kafka-broker-default is labeled with app.kubernetes.io/name: listener???
84+
# So we have a dedicated config for it
85+
apiVersion: monitoring.coreos.com/v1
86+
kind: ServiceMonitor
87+
metadata:
88+
name: stackable-kafka
89+
labels:
90+
stackable.tech/vendor: Stackable
91+
release: prometheus
92+
spec:
1593
namespaceSelector:
1694
any: true
95+
selector:
96+
matchLabels:
97+
stackable.tech/vendor: Stackable
98+
app.kubernetes.io/name: listener # Dafuq?
99+
app.kubernetes.io/component: broker # We need to filter on brokers instead, as the app.kubernetes.io/name is messed up
100+
endpoints:
101+
- scheme: http
102+
port: metrics
103+
path: /metrics
17104
jobLabel: app.kubernetes.io/instance
18105
---
106+
# We prefer the native metrics over the statsd-exporter
19107
apiVersion: monitoring.coreos.com/v1
20108
kind: ServiceMonitor
21109
metadata:
22-
name: scrape-minio
110+
name: stackable-hdfs
23111
labels:
24112
stackable.tech/vendor: Stackable
25113
release: prometheus
26114
spec:
115+
namespaceSelector:
116+
any: true
117+
selector:
118+
matchLabels:
119+
stackable.tech/vendor: Stackable
120+
prometheus.io/scrape: "true"
121+
app.kubernetes.io/name: hdfs
27122
endpoints:
28-
- path: /minio/v2/metrics/node
29-
port: http
30-
scheme: http
31-
- path: /minio/v2/metrics/cluster
32-
port: http
33-
scheme: http
123+
- scheme: http
124+
port: http # Don't use metrics!
125+
path: /prom
126+
jobLabel: app.kubernetes.io/instance
127+
---
128+
apiVersion: monitoring.coreos.com/v1
129+
kind: ServiceMonitor
130+
metadata:
131+
name: stackable-hbase
132+
labels:
133+
stackable.tech/vendor: Stackable
134+
release: prometheus
135+
spec:
136+
namespaceSelector:
137+
any: true
34138
selector:
35139
matchLabels:
140+
stackable.tech/vendor: Stackable
141+
prometheus.io/scrape: "true"
142+
app.kubernetes.io/name: hbase
143+
endpoints:
144+
- scheme: http
145+
port: ui-http
146+
path: /prometheus
147+
jobLabel: app.kubernetes.io/instance
148+
---
149+
# NiFI 2 is a beast on it's own....
150+
# The current state produces "server returned HTTP status 401 Unauthorized", but it's at least a
151+
# good starting point for a working version
152+
apiVersion: monitoring.coreos.com/v1
153+
kind: ServiceMonitor
154+
metadata:
155+
name: stackable-nifi-2
156+
labels:
157+
stackable.tech/vendor: Stackable
158+
release: prometheus
159+
spec:
160+
namespaceSelector:
161+
any: true
162+
selector:
163+
matchLabels:
164+
stackable.tech/vendor: Stackable
165+
prometheus.io/scrape: "true"
166+
matchExpressions:
167+
- key: app.kubernetes.io/name
168+
operator: In
169+
values:
170+
- nifi
171+
# - key: app.kubernetes.io/version
172+
# operator: NotIn
173+
# values:
174+
# - List all 1.x.x version combinations
175+
endpoints:
176+
- scheme: https
177+
port: https
178+
path: /nifi-api/flow/metrics/prometheus
179+
# TODO: Use mTLS
180+
# See https://github.com/stackabletech/demos/pull/260
181+
# See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#monitoring.coreos.com/v1.TLSConfig
182+
tlsConfig:
183+
insecureSkipVerify: true
184+
relabelings:
185+
- sourceLabels:
186+
- __meta_kubernetes_pod_name
187+
- __meta_kubernetes_service_name
188+
- __meta_kubernetes_namespace
189+
- __meta_kubernetes_pod_container_port_number
190+
targetLabel: __address__
191+
replacement: ${1}.${2}-headless.${3}.svc.cluster.local:${4}
192+
regex: (.+);(.+?)(?:-metrics)?;(.+);(.+)
193+
jobLabel: app.kubernetes.io/instance
194+
---
195+
apiVersion: monitoring.coreos.com/v1
196+
kind: ServiceMonitor
197+
metadata:
198+
name: stackable-minio-http
199+
labels:
200+
stackable.tech/vendor: Stackable
201+
release: prometheus
202+
spec:
203+
namespaceSelector:
204+
any: true
205+
selector:
206+
matchLabels:
207+
stackable.tech/vendor: Stackable
36208
app: minio
37209
monitoring: "true"
210+
endpoints:
211+
- scheme: http
212+
port: http
213+
path: /minio/v2/metrics/cluster
214+
---
215+
apiVersion: monitoring.coreos.com/v1
216+
kind: ServiceMonitor
217+
metadata:
218+
name: stackable-minio-https
219+
labels:
220+
stackable.tech/vendor: Stackable
221+
release: prometheus
222+
spec:
38223
namespaceSelector:
39224
any: true
225+
selector:
226+
matchLabels:
227+
stackable.tech/vendor: Stackable
228+
app: minio
229+
monitoring: "true"
230+
endpoints:
231+
- scheme: https
232+
port: https
233+
path: /minio/v2/metrics/cluster
234+
# Prevent "tls: failed to verify certificate: x509: cannot validate certificate for 100.96.234.154 because it doesn't contain any IP SANs"
235+
tlsConfig:
236+
insecureSkipVerify: true

0 commit comments

Comments
 (0)