Disable KubePersistentVolumeFillingUp for prometheus (#342)

Author: kurlov

resources/mixins/kubernetes/generated/alerts.yml

@@ -69,8 +69,9 @@ kubernetes {
                 function(rule)
                   if rule.alert == 'KubePersistentVolumeFillingUp' && rule.labels.severity == 'warning' then
                     rule {
-                      // Exclude scanner-db PVCs from warning alerts because scanner-db dynamically grows/shrinks.
-                      expr: std.rstripChars(rule.expr, '\n') + '\nunless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"scanner-db.*"}\n' % $._config,
+                      // Exclude scanner-db and prometheus PVCs from warning alerts.
+                      // scanner-db and prometheus PVC dynamically grow/shrink. They might fluctuate around warning threshold.
+                      expr: std.rstripChars(rule.expr, '\n') + '\nunless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"scanner-db.*|prometheus-data-prometheus-k8s.*"}\n' % $._config,
                     }
                   else
                     rule,

resources/mixins/kubernetes/mixin.libsonnet

@@ -422,7 +422,7 @@ spec:
             kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1
             unless on(cluster, namespace, persistentvolumeclaim)
             kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1
-            unless on(cluster, namespace, persistentvolumeclaim) kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"scanner-db.*"}
+            unless on(cluster, namespace, persistentvolumeclaim) kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"scanner-db.*|prometheus-data-prometheus-k8s.*"}
           "for": "1h"
           "labels":
             "severity": "warning"