Update NGFHigh5xxRate alert to exclude NGINX synthesized 503s from scaled-down services

Author: jb3

kubernetes/namespaces/loki/loki/alert_rules.yml

@@ -191,13 +191,14 @@ data:
 
           - alert: NGFHigh5xxRate
             expr: |
+              # NGINX synthesised 503s are not counted as they are normally from services we have set scaled to 0 and know about
               (
                 sum by (host) (
                   rate(
                     {namespace="nginx-gateway", app_container="/nginx"}
                     | json
                     | __error__ = ""
-                    | host !~ "forms-api.*"
+                    | upstream_addr!="unix:/var/run/nginx/nginx-503-server.sock"
                     | status >= 500
                     [5m]
                   )
@@ -208,7 +209,7 @@ data:
                     {namespace="nginx-gateway", app_container="/nginx"}
                     | json
                     | __error__ = ""
-                    | host !~ "forms-api.*"
+                    upstream_addr!="unix:/var/run/nginx/nginx-503-server.sock"
                     [5m]
                   )
                 )