fix(etcd-shield): raise thresholds, add total_size check, and fix severity — production

Author: peet-rh

components/etcd-shield/rings/ring-0/base/base-snapshot/etcd_shield_alerts.yaml

@@ -18,8 +18,10 @@ spec:
         description: Etcd is nearing capacity limits, so etcd-shield is denying admission
     - record: etcd_shield_trigger
       expr: |
-        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1) or
-            ((((etcd_mvcc_db_total_size_in_use_in_bytes) >= bool (etcd_server_quota_backend_bytes * 0.70)) == 1) and
+        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.85)) == 1) or
+            ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.90)) == 1) or
+            ((((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.75)) == 1) or
+                ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1)) and
                 (count without (alertname, alertstate, severity)
                       (ALERTS{
                         alertname="EtcdShieldDenyAdmission",

components/etcd-shield/rings/ring-1/base/base-snapshot/etcd_shield_alerts.yaml

@@ -18,8 +18,10 @@ spec:
         description: Etcd is nearing capacity limits, so etcd-shield is denying admission
     - record: etcd_shield_trigger
       expr: |
-        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1) or
-            ((((etcd_mvcc_db_total_size_in_use_in_bytes) >= bool (etcd_server_quota_backend_bytes * 0.70)) == 1) and
+        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.85)) == 1) or
+            ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.90)) == 1) or
+            ((((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.75)) == 1) or
+                ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1)) and
                 (count without (alertname, alertstate, severity)
                       (ALERTS{
                         alertname="EtcdShieldDenyAdmission",

components/etcd-shield/rings/ring-2/base/base-snapshot/etcd_shield_alerts.yaml

@@ -18,8 +18,10 @@ spec:
         description: Etcd is nearing capacity limits, so etcd-shield is denying admission
     - record: etcd_shield_trigger
       expr: |
-        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1) or
-            ((((etcd_mvcc_db_total_size_in_use_in_bytes) >= bool (etcd_server_quota_backend_bytes * 0.70)) == 1) and
+        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.85)) == 1) or
+            ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.90)) == 1) or
+            ((((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.75)) == 1) or
+                ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1)) and
                 (count without (alertname, alertstate, severity)
                       (ALERTS{
                         alertname="EtcdShieldDenyAdmission",

components/etcd-shield/rings/ring-3/base/base-snapshot/etcd_shield_alerts.yaml

@@ -18,8 +18,10 @@ spec:
         description: Etcd is nearing capacity limits, so etcd-shield is denying admission
     - record: etcd_shield_trigger
       expr: |
-        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1) or
-            ((((etcd_mvcc_db_total_size_in_use_in_bytes) >= bool (etcd_server_quota_backend_bytes * 0.70)) == 1) and
+        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.85)) == 1) or
+            ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.90)) == 1) or
+            ((((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.75)) == 1) or
+                ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1)) and
                 (count without (alertname, alertstate, severity)
                       (ALERTS{
                         alertname="EtcdShieldDenyAdmission",

components/etcd-shield/rings/ring-4/base/base-snapshot/etcd_shield_alerts.yaml

@@ -18,8 +18,10 @@ spec:
         description: Etcd is nearing capacity limits, so etcd-shield is denying admission
     - record: etcd_shield_trigger
       expr: |
-        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1) or
-            ((((etcd_mvcc_db_total_size_in_use_in_bytes) >= bool (etcd_server_quota_backend_bytes * 0.70)) == 1) and
+        (((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.85)) == 1) or
+            ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.90)) == 1) or
+            ((((etcd_mvcc_db_total_size_in_use_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.75)) == 1) or
+                ((etcd_mvcc_db_total_size_in_bytes >= bool (etcd_server_quota_backend_bytes * 0.80)) == 1)) and
                 (count without (alertname, alertstate, severity)
                       (ALERTS{
                         alertname="EtcdShieldDenyAdmission",