Skip to content

Commit 688ebf6

Browse files
sbernauersbernauer
committed
Scrape NiFi 2 using mTLS
1 parent 7582831 commit 688ebf6

4 files changed

Lines changed: 165 additions & 31 deletions

File tree

‎stacks/monitoring/create-prometheus-tls-certificate-serviceaccount.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
name: create-prometheus-tls-certificate-serviceaccount
6+
# ---
7+
# apiVersion: rbac.authorization.k8s.io/v1
8+
# kind: ClusterRoleBinding
9+
# metadata:
10+
# name: create-prometheus-tls-certificate-clusterrolebinding
11+
# subjects:
12+
# - kind: ServiceAccount
13+
# name: create-prometheus-tls-certificate-serviceaccount
14+
# namespace: {{ NAMESPACE }}
15+
# roleRef:
16+
# kind: ClusterRole
17+
# name: create-prometheus-tls-certificate-clusterrole
18+
# apiGroup: rbac.authorization.k8s.io
19+
# ---
20+
# apiVersion: rbac.authorization.k8s.io/v1
21+
# kind: ClusterRole
22+
# metadata:
23+
# name: create-prometheus-tls-certificate-clusterrole
24+
# rules:
25+
# - apiGroups:
26+
# - ""
27+
# resources:
28+
# - secret
29+
# verbs:
30+
# - create
31+
# - patch
32+
---
33+
apiVersion: rbac.authorization.k8s.io/v1
34+
kind: RoleBinding
35+
metadata:
36+
name: create-prometheus-tls-certificate-rolebinding
37+
subjects:
38+
- kind: ServiceAccount
39+
name: create-prometheus-tls-certificate-serviceaccount
40+
namespace: {{ NAMESPACE }}
41+
roleRef:
42+
kind: Role
43+
name: create-prometheus-tls-certificate-role
44+
apiGroup: rbac.authorization.k8s.io
45+
---
46+
apiVersion: rbac.authorization.k8s.io/v1
47+
kind: Role
48+
metadata:
49+
name: create-prometheus-tls-certificate-role
50+
rules:
51+
- apiGroups: [""]
52+
resources: ["secrets"]
53+
verbs: ["get", "create", "patch"]
54+
- apiGroups: [""]
55+
resources: ["pods"]
56+
verbs: ["delete"]

‎stacks/monitoring/create-prometheus-tls-certificate.yaml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
---
2+
apiVersion: apps/v1
3+
kind: Deployment
4+
metadata:
5+
name: create-prometheus-tls-certificate
6+
labels:
7+
app: create-prometheus-tls-certificate
8+
spec:
9+
replicas: 1
10+
selector:
11+
matchLabels:
12+
app: create-prometheus-tls-certificate
13+
template:
14+
metadata:
15+
labels:
16+
app: create-prometheus-tls-certificate
17+
spec:
18+
serviceAccountName: create-prometheus-tls-certificate-serviceaccount
19+
containers:
20+
- name: create-prometheus-tls-certificate
21+
image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
22+
env:
23+
- name: POD_NAME
24+
valueFrom:
25+
fieldRef:
26+
fieldPath: metadata.name
27+
- name: POD_NAMESPACE
28+
valueFrom:
29+
fieldRef:
30+
fieldPath: metadata.namespace
31+
command:
32+
- bash
33+
- -euo
34+
- pipefail
35+
- -c
36+
- |
37+
# "kubectl create secret" fails on existing Secrets, so we "kubectl apply" instead
38+
kubectl create secret generic prometheus-tls-certificate \
39+
--from-file=/prometheus-tls-certificate/ca.crt \
40+
--from-file=/prometheus-tls-certificate/tls.crt \
41+
--from-file=/prometheus-tls-certificate/tls.key \
42+
--dry-run=client -o yaml \
43+
| kubectl apply -f -
44+
45+
echo Sleeping 6 hours before deleting my own Pod
46+
sleep 21600 # 6 * 60 * 60
47+
48+
echo "Deleting our own Pod, so that it gets re-created and secret-operator issues a new certificate (only crash-looping the container is not enough!)"
49+
kubectl --namespace "$POD_NAMESPACE" delete pod "$POD_NAME"
50+
exit 0
51+
volumeMounts:
52+
- name: prometheus-tls-certificate
53+
mountPath: /prometheus-tls-certificate
54+
volumes:
55+
- name: prometheus-tls-certificate
56+
ephemeral:
57+
volumeClaimTemplate:
58+
metadata:
59+
annotations:
60+
# secrets.stackable.tech/backend.autotls.cert.lifetime: "1d"
61+
secrets.stackable.tech/class: "tls"
62+
secrets.stackable.tech/format: "tls-pem"
63+
secrets.stackable.tech/scope: "service=prometheus"
64+
spec:
65+
accessModes:
66+
- ReadWriteOnce
67+
resources:
68+
requests:
69+
storage: "1"
70+
storageClassName: secrets.stackable.tech
71+
volumeMode: Filesystem
72+
securityContext:
73+
fsGroup: 1000

‎stacks/monitoring/prometheus-service-monitors.yaml‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,20 @@
1-
# ### Products
2-
#
31
# Use something like this to check for metrics:
42
# count by (app_kubernetes_io_name, app_kubernetes_io_instance, pod) ({app_kubernetes_io_name!="",pod!=""})
53
#
6-
# See https://github.com/stackabletech/issues/issues/735
7-
#
8-
# - [x] Airflow - exporter
9-
# - [x] Druid - native
10-
# - [x] HBase - native
11-
# - [x] Hadoop HDFS - native
12-
# - [x] Hive - exporter
13-
# - [x] Kafka - exporter
14-
# - [x] NiFi 1 - native
15-
# - [ ] NiFi 2 - native - partially working, needs mTLS
16-
# - [ ] OpenSearch
17-
# - [ ] Spark - native
18-
# - [x] Superset - exporter
19-
# - [x] Trino - native
20-
# - [x] ZooKeeper - native
21-
# - [x] OPA - native
4+
# [x] Airflow - exporter
5+
# [x] Druid - native
6+
# [x] HBase - native
7+
# [x] Hadoop HDFS - native
8+
# [x] Hive - exporter
9+
# [x] Kafka - exporter
10+
# [x] NiFi 1 - native
11+
# [x] NiFi 2 - native
12+
# [ ] OpenSearch - not officially part of the platform yet
13+
# [ ] Spark - native - was too lazy, no idea if we even expose metrics
14+
# [x] Superset - exporter
15+
# [x] Trino - native
16+
# [x] ZooKeeper - native
17+
# [x] OPA - native
2218
---
2319
apiVersion: monitoring.coreos.com/v1
2420
kind: ServiceMonitor
@@ -171,9 +167,8 @@ spec:
171167
- app.kubernetes.io/role-group
172168
- app.kubernetes.io/version
173169
---
174-
# NiFI 2 is a beast on it's own....
175-
# The current state produces "server returned HTTP status 401 Unauthorized", but it's at least a
176-
# good starting point for a working version
170+
# NiFI 2 is a beast on it's own...
171+
# We need to use mTLS (otherwise we get a 401) and can not use the PodIP
177172
apiVersion: monitoring.coreos.com/v1
178173
kind: ServiceMonitor
179174
metadata:
@@ -193,19 +188,26 @@ spec:
193188
operator: In
194189
values:
195190
- nifi
196-
# - key: app.kubernetes.io/version
197-
# operator: NotIn
198-
# values:
199-
# - List all 1.x.x version combinations
200191
endpoints:
201192
- scheme: https
202193
port: https
203194
path: /nifi-api/flow/metrics/prometheus
204-
# TODO: Use mTLS
205-
# See https://github.com/stackabletech/demos/pull/260
206195
# See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#monitoring.coreos.com/v1.TLSConfig
207196
tlsConfig:
208-
insecureSkipVerify: true
197+
ca:
198+
secret:
199+
name: prometheus-tls-certificate
200+
key: ca.crt
201+
cert:
202+
secret:
203+
name: prometheus-tls-certificate
204+
key: tls.crt
205+
keySecret:
206+
name: prometheus-tls-certificate
207+
key: tls.key
208+
# We need to talk to the Pod via the FQDN of the Pod because of the stupid SNI check of NiFi.
209+
# We can not use the typical PodIP, as it is not contained in the NiFi certificate,
210+
# see https://github.com/stackabletech/secret-operator/issues/620
209211
relabelings:
210212
- sourceLabels:
211213
- __meta_kubernetes_pod_name

‎stacks/stacks-v2.yaml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,17 @@ stacks:
66
stackableOperators:
77
- commons
88
- listener
9+
- secret
910
labels:
1011
- monitoring
1112
- prometheus
1213
- grafana
1314
manifests:
14-
- plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/grafana-dashboards.yaml
15-
- helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/prometheus.yaml
16-
- plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/prometheus-service-monitors.yaml
15+
- plainYaml: stacks/monitoring/grafana-dashboards.yaml
16+
- plainYaml: stacks/monitoring/create-prometheus-tls-certificate-serviceaccount.yaml
17+
- plainYaml: stacks/monitoring/create-prometheus-tls-certificate.yaml
18+
- helmChart: stacks/monitoring/prometheus.yaml
19+
- plainYaml: stacks/monitoring/prometheus-service-monitors.yaml
1720
supportedNamespaces: []
1821
resourceRequests:
1922
cpu: 1750m

0 commit comments

Comments
 (0)