ROX-33228: Disable CPUThrottlingHigh alerts for config-reloader (#338)

Author: kovayur

resources/mixins/kubernetes/generated/alerts.yml

@@ -49,6 +49,20 @@ kubernetes {
                 group.rules
               ),
             }
+          else if group.name == 'kubernetes-resources' then
+            group {
+              rules: std.map(
+                function(rule)
+                  if rule.alert == 'CPUThrottlingHigh' then
+                    rule {
+                      // Exclude config-reloader in rhacs-observability namespace from false positive CPU throttling alerts.
+                      expr: std.rstripChars(rule.expr, '\n') + '\nunless on(%(clusterLabel)s, container, pod, namespace) container_cpu_cfs_throttled_periods_total{namespace="rhacs-observability", container="config-reloader"}\n' % $._config,
+                    }
+                  else
+                    rule,
+                group.rules
+              ),
+            }
           else if group.name == 'kubernetes-system-kubelet' then
             group {
               rules: std.filter(

resources/mixins/kubernetes/mixin.libsonnet

@@ -375,6 +375,7 @@ spec:
               /
             sum(increase(container_cpu_cfs_periods_total{namespace!~"openshift-.*"}[5m])) by (cluster, container, pod, namespace)
               > ( 25 / 100 )
+            unless on(cluster, container, pod, namespace) container_cpu_cfs_throttled_periods_total{namespace="rhacs-observability", container="config-reloader"}
           "for": "15m"
           "labels":
             "severity": "info"