Merge pull request #324 from stackrox/ebenshet/tenant-memory-alerts-shhh

Reduce RHACSOperatorMemoryUtilizationCritical from critical to warning

Author: ebensh

resources/prometheus/prometheus-rules.yaml

@@ -260,17 +260,6 @@ spec:
             summary: tenant '{{ $labels.namespace }}' container '{{ $labels.container }}' in pod '{{ $labels.pod }}' is reaching its memory limit.
             description: tenant '{{ $labels.namespace }}' container '{{ $labels.container }}' in pod '{{ $labels.pod }}' reached {{ $value | humanizePercentage }} of its memory limit and is at risk of being OOM killed.
             sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-039-tenant-workload-memory-utilization-high.md"
-        - alert: RHACSTenantWorkloadMemoryUtilizationCritical
-          expr: |
-            rhacs_tenants:namespace:pod:container:max_memory_usage_ratio{container=~"central|scanner|indexer|matcher"} >= 0.95
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: tenant '{{ $labels.namespace }}' container '{{ $labels.container }}' in pod '{{ $labels.pod }}' is critically reaching its memory limit.
-            description: tenant '{{ $labels.namespace }}' container '{{ $labels.container }}' in pod '{{ $labels.pod }}' reached {{ $value | humanizePercentage }} of its memory limit and is at high risk of being OOM killed.
-            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-039-tenant-workload-memory-utilization-high.md"
-
     - name: rhacs-operator
       rules:
         - expr: |

resources/prometheus/unit_tests/RHACSTenantWorkloadMemoryUtilizationHigh.yaml

@@ -28,26 +28,3 @@ tests:
               summary: tenant 'rhacs-aaaaaaaaaaaaaaaaaaaa' container 'central' in pod 'mypod' is reaching its memory limit.
               description: tenant 'rhacs-aaaaaaaaaaaaaaaaaaaa' container 'central' in pod 'mypod' reached 90% of its memory limit and is at risk of being OOM killed.
               sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-039-tenant-workload-memory-utilization-high.md"
-  - interval: 1m
-    input_series:
-      - series: container_memory_working_set_bytes{namespace="rhacs-aaaaaaaaaaaaaaaaaaaa",pod="mypod",container="central"}
-        values: "50+0x10 95+0x10"
-      - series: container_spec_memory_limit_bytes{namespace="rhacs-aaaaaaaaaaaaaaaaaaaa",pod="mypod",container="central"}
-        values: "100+0x20"
-    alert_rule_test:
-      - eval_time: 1m
-        alertname: RHACSTenantWorkloadMemoryUtilizationCritical
-        exp_alerts: []
-      - eval_time: 21m
-        alertname: RHACSTenantWorkloadMemoryUtilizationCritical
-        exp_alerts:
-          - exp_labels:
-              alertname: RHACSTenantWorkloadMemoryUtilizationCritical
-              severity: critical
-              namespace: rhacs-aaaaaaaaaaaaaaaaaaaa
-              pod: mypod
-              container: central
-            exp_annotations:
-              description: tenant 'rhacs-aaaaaaaaaaaaaaaaaaaa' container 'central' in pod 'mypod' reached 95% of its memory limit and is at high risk of being OOM killed.
-              summary: tenant 'rhacs-aaaaaaaaaaaaaaaaaaaa' container 'central' in pod 'mypod' is critically reaching its memory limit.
-              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-039-tenant-workload-memory-utilization-high.md"