Skip to content

Commit 70f2d20

Browse files
pablin-10pablin-10
authored
Prometheus metrics setup (#125)
1 parent 60561c7 commit 70f2d20

16 files changed

Lines changed: 346 additions & 3 deletions

File tree

‎images/flashbox-l1.conf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
Include=shared/mkosi.conf
33
Include=modules/flashbox/common/mkosi.conf
44
Include=modules/flashbox/flashbox-l1/mkosi.conf
5+
Include=modules/flashbox/observability/mkosi.conf
56

67
[Config]
78
Profiles=azure,gcp

‎modules/flashbox/common/mkosi.extra/usr/bin/init-container.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ SEARCHER_SSH_PORT=10022
88
SEARCHER_INPUT_UDP_PORT=27017 # External UDP input channel
99
SEARCHER_INPUT_TCP_PORT=27018 # External TCP input channel
1010

11+
# ENDPOINTS
12+
PROMETHEUS_PROXY_IP=10.88.0.100 # host firewall allows in ALWAYS_OUT
13+
1114
# Run extra commands which are customized per image,
1215
# see bob*/mkosi.extra/etc/bob/searcher-container-before-init
1316
#
@@ -73,6 +76,9 @@ ns_iptables() {
7376

7477
ns_iptables -A OUTPUT -d 169.254.169.254 -j DROP
7578

79+
# Block container from reaching the internal Prometheus Proxy
80+
ns_iptables -A OUTPUT -d $PROMETHEUS_PROXY_IP -j DROP
81+
7682
# Block consensus layer P2P port (TCP and UDP)
7783
ns_iptables -A OUTPUT -p tcp --dport 9000 -j DROP
7884
ns_iptables -A OUTPUT -p udp --dport 9000 -j DROP

‎modules/flashbox/common/mkosi.extra/usr/bin/init-firewall.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ drop_dst_ip() {
151151
#
152152
# `source` is not supported in dash
153153
###########################################################################
154+
154155
. /etc/bob/firewall-config
155156

156157
###########################################################################

‎modules/flashbox/common/mkosi.extra/usr/bin/toggle‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ CHAIN_MAINTENANCE_OUT="MAINTENANCE_OUT"
4545
CHAIN_PRODUCTION_IN="PRODUCTION_IN"
4646
CHAIN_PRODUCTION_OUT="PRODUCTION_OUT"
4747

48+
# Endpoints
49+
PROMETHEUS_PROXY_IP="10.88.0.100"
50+
4851
# Source image-specific configuration if available
4952
TOGGLE_CONFIG="/etc/bob/toggle-config"
5053
if [ -f "$TOGGLE_CONFIG" ]; then
@@ -217,11 +220,13 @@ check_searcher_namespace_rules() {
217220
&& nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --sport 27017 -j DROP 2>/dev/null \
218221
&& nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --sport 27018 -j DROP 2>/dev/null \
219222
&& nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p udp --dport 123 -j DROP 2>/dev/null \
220-
&& nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --dport 123 -j DROP 2>/dev/null; then
221-
echo "OK: searcher-container firewall rules (DROP tcp/udp dport 9000/123, tcp/udp sport 27017, tcp sport 27018) are ACTIVE."
223+
&& nsenter --target "$pid" --net $IPTABLES -C OUTPUT -p tcp --dport 123 -j DROP 2>/dev/null \
224+
&& nsenter --target "$pid" --net $IPTABLES -C OUTPUT -d 169.254.169.254 -j DROP 2>/dev/null \
225+
&& nsenter --target "$pid" --net $IPTABLES -C OUTPUT -d $PROMETHEUS_PROXY_IP -j DROP 2>/dev/null; then
226+
echo "OK: searcher-container firewall rules (DROP tcp/udp dport 9000/123, tcp/udp sport 27017, tcp sport 27018, GCE metadata 169.254.169.254, metrics proxy 10.88.0.100) are ACTIVE."
222227
return 0
223228
else
224-
echo "WARNING: One or more searcher-container firewall rules (DROP ports 9000/27017/27018/123) are NOT found."
229+
echo "WARNING: One or more searcher-container firewall rules (DROP ports 9000/27017/27018/123, GCE metadata 169.254.169.254, metrics proxy 10.88.0.100) are NOT found."
225230
return 1
226231
fi
227232
}

‎modules/flashbox/flashbox-l1/mkosi.extra/etc/bob/firewall-config‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ FLASHBOTS_BUNDLE_2="3.15.88.156"
3232
FLASHBOTS_TX_STREAM_1="3.136.107.142"
3333
FLASHBOTS_TX_STREAM_2="3.149.14.12"
3434

35+
# Prometheus metrics proxy
36+
PROMETHEUS_PROXY_IP="10.88.0.100"
37+
3538
###########################################################################
3639
# (1) ALWAYS_IN: Inbound rules that are always applied
3740
###########################################################################
@@ -74,6 +77,9 @@ accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $TITAN_IP $TITAN_BUNDLE_PORT "Titan bui
7477
# they can observe the presence or absence of packets.
7578
accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $FLASHBOTS_BUNDLE_1,$FLASHBOTS_BUNDLE_2 $HTTPS_PORT "Flashbots bundle"
7679

80+
# Prometheus metrics proxy (searcher netns has a DROP on this IP — see init-container.sh)
81+
accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp $PROMETHEUS_PROXY_IP $HTTPS_PORT "Prometheus metrics proxy"
82+
7783
###########################################################################
7884
# (3) MAINTENANCE_IN: Inbound rules for Maintenance Mode
7985
###########################################################################

‎modules/flashbox/observability/mkosi.conf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[Content]
2+
ExtraTrees=modules/flashbox/observability/mkosi.extra
3+
PostInstallationScripts=modules/flashbox/observability/mkosi.postinst
4+
5+
Packages=prometheus
6+
prometheus-node-exporter
7+
prometheus-process-exporter
8+
gettext-base

‎modules/flashbox/observability/mkosi.extra/etc/prometheus/process-exporter.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
process_names:
2+
# Monitor the searcher container (conmon + all children via --children flag)
3+
- name: "searcher-container"
4+
cmdline:
5+
- 'conmon.*searcher-container'

‎modules/flashbox/observability/mkosi.extra/etc/prometheus/prometheus.yml.tmpl‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
global:
2+
scrape_interval: 15s
3+
evaluation_interval: 15s
4+
# Stamped onto every series sent via remote_write so the upstream can
5+
# distinguish samples coming from different flashbox VMs.
6+
external_labels:
7+
host: ${FLASHBOX_VM}
8+
9+
# Recording rules for aggregated metrics
10+
rule_files:
11+
- /etc/prometheus/recording_rules.yml
12+
13+
scrape_configs:
14+
# Node exporter on localhost
15+
- job_name: 'node'
16+
static_configs:
17+
- targets: ['localhost:9100']
18+
metric_relabel_configs:
19+
# Only keep aggregated metrics for remote write
20+
- source_labels: [__name__]
21+
regex: 'node_(cpu|memory|disk|filesystem|network|vmstat)_.*'
22+
action: keep
23+
24+
# Process exporter for container monitoring
25+
- job_name: 'process'
26+
static_configs:
27+
- targets: ['localhost:9256']
28+
29+
# Remote write via the internal Prometheus metrics proxy at the fixed proxy IP,
30+
# so no DNS resolution happens on the VM; TLS SNI + cert validation are
31+
# overridden via tls_config.server_name to match the cert the proxy's upstream
32+
# presents.
33+
remote_write:
34+
- url: https://${PROMETHEUS_PROXY_IP}/workspaces/${METRICS_FLASHBOTS_WORKSPACE}/api/v1/remote_write
35+
write_relabel_configs:
36+
# Only send flashbox: prefixed metrics
37+
- source_labels: [__name__]
38+
regex: 'flashbox:.*'
39+
action: keep
40+
tls_config:
41+
server_name: aps-workspaces.${METRICS_FLASHBOTS_REGION}.amazonaws.com
42+
sigv4:
43+
region: ${METRICS_FLASHBOTS_REGION}
44+
access_key: ${METRICS_FLASHBOTS_ACCESS_KEY}
45+
secret_key: ${METRICS_FLASHBOTS_SECRET_KEY}

‎modules/flashbox/observability/mkosi.extra/etc/prometheus/recording_rules.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
groups:
2+
# Base metrics — local: prefix means they stay inside the TEE
3+
# (remote_write only forwards flashbox:*)
4+
- name: local_container_metrics
5+
interval: 30s
6+
rules:
7+
# Percent of total machine CPU capacity (normalized by core count, so the
8+
# 0-100 scale holds regardless of how many vCPUs the VM has).
9+
- record: local:container_cpu_percent
10+
expr: >
11+
sum(rate(namedprocess_namegroup_cpu_seconds_total{groupname="searcher-container"}[5m])) * 100
12+
/ count(node_cpu_seconds_total{mode="idle"})
13+
14+
# Forwarded metrics — flashbox: prefix, picked up by remote_write
15+
- name: flashbox_health
16+
interval: 30s
17+
rules:
18+
# 1 if at least one process is running in the searcher-container group,
19+
# absent if process-exporter isn't reporting it.
20+
- record: flashbox:container_alive
21+
expr: namedprocess_namegroup_num_procs{groupname="searcher-container"} > bool 0
22+
23+
# Spike-guarded: current 15m avg must be under 80%,
24+
# AND the 10m max ending 5m ago must have been under 70%
25+
- record: flashbox:container_average_cpu_is_under_80_percent
26+
expr: >
27+
(avg_over_time(local:container_cpu_percent[15m]) < bool 80)
28+
* (max_over_time(local:container_cpu_percent[10m] offset 5m) < bool 70)
29+
30+
- record: flashbox:container_oom_kills_count
31+
expr: node_vmstat_oom_kill
32+
33+
- record: flashbox:disk_free_space_is_over_10_percent
34+
expr: >
35+
(node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) > bool 0.1
36+
37+
- record: flashbox:disk_free_space_is_over_128_gb
38+
expr: >
39+
(node_filesystem_avail_bytes{mountpoint="/persistent"}) > bool (128 * 1024 * 1024 * 1024)
40+
41+
- record: flashbox:network_is_up
42+
expr: >
43+
(sum(rate(node_network_receive_bytes_total{device!~"lo"}[5m]))
44+
+ sum(rate(node_network_transmit_bytes_total{device!~"lo"}[5m])))
45+
> bool 0

‎modules/flashbox/observability/mkosi.extra/etc/systemd/system/flashbox-observability-setup.service‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[Unit]
2+
Description=Flashbox observability setup (fetch creds, render Prometheus config)
3+
After=network-online.target
4+
Wants=network-online.target
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/bin/flashbox-observability-setup
9+
RemainAfterExit=yes
10+
StandardOutput=journal
11+
StandardError=journal
12+
13+
[Install]
14+
WantedBy=minimal.target

0 commit comments

Comments
 (0)