-
Notifications
You must be signed in to change notification settings - Fork 305
Expand file tree
/
Copy pathsilence_utils.py
More file actions
175 lines (131 loc) · 5.53 KB
/
silence_utils.py
File metadata and controls
175 lines (131 loc) · 5.53 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
import json
import logging
from datetime import datetime
from enum import Enum
from typing import Dict, List, Optional
from uuid import UUID
import requests
from pydantic import BaseModel, SecretStr, validator
from robusta.core.exceptions import AlertsManagerNotFound, NoAlertManagerUrlFound
from robusta.core.model.base_params import ActionParams
from robusta.integrations import openshift
from robusta.integrations.prometheus.utils import AlertManagerDiscovery, ServiceDiscovery
# ref to api https://github.com/prometheus/alertmanager/blob/main/api/v2/openapi.yaml
class Matcher(BaseModel):
# https://github.com/prometheus/alertmanager/blob/main/api/v2/models/matcher.go
isEqual: bool = (
True # support old version matchers with omitted isEqual https://github.com/prometheus/alertmanager/pull/2603
)
isRegex: bool
name: str
value: str
class SilenceStatus(BaseModel):
# https://github.com/prometheus/alertmanager/blob/main/api/v2/models/silence_status.go
state: str
class Silence(BaseModel):
# https://github.com/prometheus/alertmanager/blob/main/api/v2/models/silence.go
id: UUID
status: SilenceStatus
comment: str
createdBy: str
startsAt: datetime
endsAt: datetime
matchers: List[Matcher]
def to_list(self) -> List[str]:
return [
str(self.id),
self.status.json(),
self.comment,
self.createdBy,
self.startsAt.isoformat(timespec="seconds"),
self.endsAt.isoformat(timespec="seconds"),
json.dumps([matcher.dict() for matcher in self.matchers]),
]
class AlertManagerParams(ActionParams):
"""
:var alertmanager_url: Alternative Alert Manager url to send requests.
:var alertmanager_additional_headers: additional HTTP headers (if defined) to add to every alertmanager request
"""
alertmanager_flavor: str = None # type: ignore
alertmanager_url: Optional[str]
alertmanager_auth: Optional[SecretStr] = None
alertmanager_additional_headers: Optional[Dict[str, str]] = None
grafana_api_key: str = None # type: ignore
@validator("alertmanager_url", allow_reuse=True)
def validate_alertmanager_url(cls, v):
if v and not v.startswith("http"): # if the user configured url without http(s)
v = f"http://{v}"
logging.info(f"Adding protocol to alertmanager_url: {v}")
if v.endswith("/"):
return v[:-1]
return v
@validator("alertmanager_auth", allow_reuse=True, always=True)
def auto_openshift_token(cls, v: Optional[SecretStr]):
# If openshift is enabled, and the user didn't configure alertmanager_auth, we will try to load the token from the service account
if v is not None:
return v
openshift_token = openshift.load_token()
if openshift_token is not None:
logging.debug(f"Using openshift token from {openshift.TOKEN_LOCATION} for alertmanager auth")
return SecretStr(f"Bearer {openshift_token}")
return None
class DeleteSilenceParams(AlertManagerParams):
"""
:var id: uuid of the silence.
"""
id: str
class AddSilenceParams(AlertManagerParams):
"""
:var id: uuid of the silence. use for update, empty on create.
:var comment: text comment of the silence.
:var createdBy: author of the silence.
:var startsAt: date.
:var endsAt: date.
:var matchers: List of matchers to filter the silence effect.
"""
id: Optional[str]
comment: str
createdBy: str
startsAt: datetime
endsAt: datetime
matchers: List[Matcher]
def get_alertmanager_silences_connection(params: AlertManagerParams):
alertmanager_url = get_alertmanager_url(params)
if not alertmanager_url:
raise NoAlertManagerUrlFound("AlertManager url could not be found. Add 'alertmanager_url' under global_config")
try:
response = requests.get(
f"{alertmanager_url}{get_alertmanager_url_path(SilenceOperation.LIST, params)}",
headers=gen_alertmanager_headers(params),
)
response.raise_for_status()
except Exception as e:
raise AlertsManagerNotFound(
f"Could not connect to the alert manager [{alertmanager_url}] \nCaused by {e.__class__.__name__}: {e})"
) from e
SilenceOperation = Enum("SilenceOperation", "CREATE DELETE LIST")
def gen_alertmanager_headers(params: AlertManagerParams) -> Dict:
headers = {"Content-type": "application/json"}
if params.grafana_api_key:
headers.update({"Authorization": f"Bearer {params.grafana_api_key}"})
elif params.alertmanager_auth:
headers.update({"Authorization": params.alertmanager_auth.get_secret_value()})
if params.alertmanager_additional_headers:
headers.update(params.alertmanager_additional_headers)
return headers
def get_alertmanager_url_path(operation: SilenceOperation, params: AlertManagerParams) -> str:
prefix = ""
if "grafana" == params.alertmanager_flavor:
prefix = "/api/alertmanager/grafana"
if operation == SilenceOperation.DELETE:
return f"{prefix}/api/v2/silence"
else:
return f"{prefix}/api/v2/silences"
def get_alertmanager_url(params: AlertManagerParams) -> str:
if params.alertmanager_url:
return params.alertmanager_url
if "grafana" == params.alertmanager_flavor:
return ServiceDiscovery.find_url(
selectors=["app.kubernetes.io/name=grafana"], error_msg="Failed to find grafana url"
)
return AlertManagerDiscovery.find_alert_manager_url()