Skip to content

Commit 1e0a9ef

Browse files
jb3jb3
committed
Add new recording rules for NGF monitoring in Loki
1 parent 22bfb71 commit 1e0a9ef

1 file changed

Lines changed: 276 additions & 0 deletions

File tree

‎kubernetes/namespaces/loki/recording_rules.yml‎

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: loki-recording-rules
5+
namespace: loki
6+
labels:
7+
loki_rule: 'yes'
8+
data:
9+
loki-recording-rules.yaml: |
10+
groups:
11+
12+
# ------------------------------------------------------------
13+
# Traffic gate - reusable minimum-traffic threshold
14+
# ------------------------------------------------------------
15+
- name: ngf_traffic_gate
16+
interval: 1m
17+
rules:
18+
- record: ngf:requests:rate5m
19+
expr: |
20+
sum by (host) (
21+
rate(
22+
{namespace="nginx-gateway", app_container="/nginx"}
23+
| json
24+
| __error__ = ""
25+
[5m]
26+
)
27+
)
28+
29+
# ------------------------------------------------------------
30+
# Request rate - replaces nginx_ingress_controller_requests
31+
# ------------------------------------------------------------
32+
- name: ngf_request_rates
33+
interval: 1m
34+
rules:
35+
36+
- record: ngf:requests:rate2m
37+
expr: |
38+
sum by (host, method, status) (
39+
rate(
40+
{namespace="nginx-gateway", app_container="/nginx"}
41+
| json
42+
| __error__ = ""
43+
[2m]
44+
)
45+
)
46+
47+
- record: ngf:requests_5xx:rate2m
48+
expr: |
49+
sum by (host) (
50+
rate(
51+
{namespace="nginx-gateway", app_container="/nginx"}
52+
| json
53+
| __error__ = ""
54+
| status >= 500
55+
[2m]
56+
)
57+
)
58+
59+
- record: ngf:requests_4xx:rate2m
60+
expr: |
61+
sum by (host) (
62+
rate(
63+
{namespace="nginx-gateway", app_container="/nginx"}
64+
| json
65+
| __error__ = ""
66+
| status >= 400, status < 500
67+
[2m]
68+
)
69+
)
70+
71+
- record: ngf:upstream_errors:rate2m
72+
expr: |
73+
sum by (host, upstream_addr) (
74+
rate(
75+
{namespace="nginx-gateway", app_container="/nginx"}
76+
| json
77+
| __error__ = ""
78+
| upstream_status =~ "[0-9]+"
79+
| upstream_status >= 500
80+
[2m]
81+
)
82+
)
83+
84+
- record: ngf:requests_by_protocol:rate2m
85+
expr: |
86+
sum by (host, protocol) (
87+
rate(
88+
{namespace="nginx-gateway", app_container="/nginx"}
89+
| json
90+
| __error__ = ""
91+
[2m]
92+
)
93+
)
94+
95+
- record: ngf:dropped_requests:rate2m
96+
expr: |
97+
sum by (host) (
98+
rate(
99+
{namespace="nginx-gateway", app_container="/nginx"}
100+
| json
101+
| __error__ = ""
102+
| request_completion != "OK"
103+
[2m]
104+
)
105+
)
106+
107+
- record: ngf:rate_limited_requests:rate2m
108+
expr: |
109+
sum by (host, limit_req_status) (
110+
rate(
111+
{namespace="nginx-gateway", app_container="/nginx"}
112+
| json
113+
| __error__ = ""
114+
| limit_req_status =~ "DELAYED|REJECTED"
115+
[2m]
116+
)
117+
)
118+
119+
# ------------------------------------------------------------
120+
# End-to-end latency (request_time)
121+
# ------------------------------------------------------------
122+
- name: ngf_request_latency
123+
interval: 1m
124+
rules:
125+
126+
- record: ngf:request_duration_p50:2m
127+
expr: |
128+
quantile_over_time(0.50,
129+
{namespace="nginx-gateway", app_container="/nginx"}
130+
| json
131+
| __error__ = ""
132+
| unwrap request_time [2m]
133+
) by (host)
134+
135+
- record: ngf:request_duration_p95:2m
136+
expr: |
137+
quantile_over_time(0.95,
138+
{namespace="nginx-gateway", app_container="/nginx"}
139+
| json
140+
| __error__ = ""
141+
| unwrap request_time [2m]
142+
) by (host)
143+
144+
- record: ngf:request_duration_p99:2m
145+
expr: |
146+
quantile_over_time(0.99,
147+
{namespace="nginx-gateway", app_container="/nginx"}
148+
| json
149+
| __error__ = ""
150+
| unwrap request_time [2m]
151+
) by (host)
152+
153+
# ------------------------------------------------------------
154+
# Upstream latency breakdown
155+
# ------------------------------------------------------------
156+
- name: ngf_upstream_latency
157+
interval: 1m
158+
rules:
159+
160+
- record: ngf:upstream_connect_time_p99:2m
161+
expr: |
162+
quantile_over_time(0.99,
163+
{namespace="nginx-gateway", app_container="/nginx"}
164+
| json
165+
| __error__ = ""
166+
| upstream_connect_time =~ "[0-9.]+"
167+
| unwrap upstream_connect_time [2m]
168+
) by (host)
169+
170+
- record: ngf:upstream_header_time_p99:2m
171+
expr: |
172+
quantile_over_time(0.99,
173+
{namespace="nginx-gateway", app_container="/nginx"}
174+
| json
175+
| __error__ = ""
176+
| upstream_header_time =~ "[0-9.]+"
177+
| unwrap upstream_header_time [2m]
178+
) by (host)
179+
180+
- record: ngf:upstream_response_time_p50:2m
181+
expr: |
182+
quantile_over_time(0.50,
183+
{namespace="nginx-gateway", app_container="/nginx"}
184+
| json
185+
| __error__ = ""
186+
| upstream_response_time =~ "[0-9.]+"
187+
| unwrap upstream_response_time [2m]
188+
) by (host)
189+
190+
- record: ngf:upstream_response_time_p95:2m
191+
expr: |
192+
quantile_over_time(0.95,
193+
{namespace="nginx-gateway", app_container="/nginx"}
194+
| json
195+
| __error__ = ""
196+
| upstream_response_time =~ "[0-9.]+"
197+
| unwrap upstream_response_time [2m]
198+
) by (host)
199+
200+
- record: ngf:upstream_response_time_p99:2m
201+
expr: |
202+
quantile_over_time(0.99,
203+
{namespace="nginx-gateway", app_container="/nginx"}
204+
| json
205+
| __error__ = ""
206+
| upstream_response_time =~ "[0-9.]+"
207+
| unwrap upstream_response_time [2m]
208+
) by (host)
209+
210+
- record: ngf:upstream_queue_time_p99:2m
211+
expr: |
212+
quantile_over_time(0.99,
213+
{namespace="nginx-gateway", app_container="/nginx"}
214+
| json
215+
| __error__ = ""
216+
| upstream_queue_time =~ "[0-9.]+"
217+
| unwrap upstream_queue_time [2m]
218+
) by (host)
219+
220+
- record: ngf:upstream_response_time_p99_by_pod:2m
221+
expr: |
222+
quantile_over_time(0.99,
223+
{namespace="nginx-gateway", app_container="/nginx"}
224+
| json
225+
| __error__ = ""
226+
| upstream_response_time =~ "[0-9.]+"
227+
| unwrap upstream_response_time [2m]
228+
) by (host, upstream_addr)
229+
230+
# ------------------------------------------------------------
231+
# Throughput
232+
# ------------------------------------------------------------
233+
- name: ngf_throughput
234+
interval: 1m
235+
rules:
236+
237+
- record: ngf:bytes_sent:rate2m
238+
expr: |
239+
sum by (host) (
240+
rate(
241+
{namespace="nginx-gateway", app_container="/nginx"}
242+
| json
243+
| __error__ = ""
244+
| unwrap bytes_sent [2m]
245+
)
246+
)
247+
248+
- record: ngf:request_size:rate2m
249+
expr: |
250+
sum by (host) (
251+
rate(
252+
{namespace="nginx-gateway", app_container="/nginx"}
253+
| json
254+
| __error__ = ""
255+
| unwrap request_length [2m]
256+
)
257+
)
258+
259+
# ------------------------------------------------------------
260+
# TLS
261+
# ------------------------------------------------------------
262+
- name: ngf_tls
263+
interval: 5m
264+
rules:
265+
266+
- record: ngf:requests_by_tls_protocol:rate5m
267+
expr: |
268+
sum by (host, ssl_protocol) (
269+
rate(
270+
{namespace="nginx-gateway", app_container="/nginx"}
271+
| json
272+
| __error__ = ""
273+
| ssl_protocol != ""
274+
[5m]
275+
)
276+
)

0 commit comments

Comments
 (0)