diff --git a/charts/cluster/prometheus_rules/cluster-logical_replication_errors-critical.yaml b/charts/cluster/prometheus_rules/cluster-logical_replication_errors-critical.yaml index 93f469bf9c..8c33caa8e0 100644 --- a/charts/cluster/prometheus_rules/cluster-logical_replication_errors-critical.yaml +++ b/charts/cluster/prometheus_rules/cluster-logical_replication_errors-critical.yaml @@ -9,7 +9,7 @@ annotations: CRITICAL: High error rate indicates persistent replication issues requiring immediate attention. This could lead to significant data inconsistency or complete replication failure. Errors include both apply errors and sync errors. The subscription may stop working if errors continue. runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/CNPGClusterLogicalReplicationErrors.md expr: | - label_replace(increase(max by (namespace, job, subname) (cnpg_pg_stat_subscription_apply_error_count + cnpg_pg_stat_subscription_sync_error_count)[5m]), "cluster", "$1", "job", ".+/(.+)") >= 5 + label_replace(max by (namespace, job, subname) (increase(cnpg_pg_stat_subscription_apply_error_count[5m]) + increase(cnpg_pg_stat_subscription_sync_error_count[5m])), "cluster", "$1", "job", ".+/(.+)") >= 5 for: 0m labels: severity: critical diff --git a/charts/cluster/prometheus_rules/cluster-logical_replication_errors.yaml b/charts/cluster/prometheus_rules/cluster-logical_replication_errors.yaml index d8987dc83d..cb39de7be6 100644 --- a/charts/cluster/prometheus_rules/cluster-logical_replication_errors.yaml +++ b/charts/cluster/prometheus_rules/cluster-logical_replication_errors.yaml @@ -9,7 +9,7 @@ annotations: This includes both apply errors (during normal replication) and sync errors (during initial table sync). Errors indicate data consistency issues that need immediate attention to prevent data divergence. runbook_url: https://github.com/cloudnative-pg/charts/blob/main/charts/cluster/docs/runbooks/{{ $alert }}.md expr: | - label_replace(increase(max by (namespace, job, subname) (cnpg_pg_stat_subscription_apply_error_count + cnpg_pg_stat_subscription_sync_error_count)[5m]), "cluster", "$1", "job", ".+/(.+)") > 0 + label_replace(max by (namespace, job, subname) (increase(cnpg_pg_stat_subscription_apply_error_count[5m]) + increase(cnpg_pg_stat_subscription_sync_error_count[5m])), "cluster", "$1", "job", ".+/(.+)") > 0 for: 1m labels: severity: warning diff --git a/charts/cluster/prometheus_rules/cluster-logical_replication_stopped-critical.yaml b/charts/cluster/prometheus_rules/cluster-logical_replication_stopped-critical.yaml index 0e2dd72418..77233c9434 100644 --- a/charts/cluster/prometheus_rules/cluster-logical_replication_stopped-critical.yaml +++ b/charts/cluster/prometheus_rules/cluster-logical_replication_stopped-critical.yaml @@ -18,7 +18,7 @@ expr: | ) or ( # Subscription is enabled but stuck (no worker process with significant lag) label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_enabled), "cluster", "$1", "job", ".+/(.+)") == 1 - and label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_pid), "cluster", "$1", "job", ".+/(.+)") == "" + and label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_pid), "cluster", "$1", "job", ".+/(.+)") == 0 and label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_buffered_lag_bytes), "cluster", "$1", "job", ".+/(.+)") / 1024^3 > 0.1 ) for: 15m diff --git a/charts/cluster/prometheus_rules/cluster-logical_replication_stopped.yaml b/charts/cluster/prometheus_rules/cluster-logical_replication_stopped.yaml index 8865ae05de..e56e0f4e5d 100644 --- a/charts/cluster/prometheus_rules/cluster-logical_replication_stopped.yaml +++ b/charts/cluster/prometheus_rules/cluster-logical_replication_stopped.yaml @@ -17,7 +17,7 @@ expr: | ) or ( # Subscription is enabled but stuck (no worker process with significant lag) label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_enabled), "cluster", "$1", "job", ".+/(.+)") == 1 - and label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_pid), "cluster", "$1", "job", ".+/(.+)") == "" + and label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_pid), "cluster", "$1", "job", ".+/(.+)") == 0 and label_replace(max by (namespace, job, subname) (cnpg_pg_stat_subscription_buffered_lag_bytes), "cluster", "$1", "job", ".+/(.+)") / 1024^3 > 0.1 ) for: 5m