Emit Warning Events for Consul Integration (DataDog#23779)

* Add new config health_check_warning_events

* Emit event for warnings

* Removing whitespace to match original message

* Add test for warning events

* Add changelog

* Fix typo

* Apply lint

* Change wording

Author: ddog-nasirthomas

consul/assets/configuration/spec.yaml

@@ -178,6 +178,16 @@ files:
         example: 3600
         minimum: 1
 
+    - name: health_check_warning_events
+      fleet_configurable: true
+      description: |
+        Whether to emit an event when a Consul health check transitions to `warning`.
+        Events for critical health checks are always emitted when `collect_health_checks` is enabled.
+      value:
+        type: boolean
+        example: true
+        default: false
+  
     - template: instances/http
     - template: instances/default
   - template: logs

consul/changelog.d/23779.added

@@ -0,0 +1 @@
+Add new config `health_check_warning_events` to emit an event when a Consul health check transitions to `warning`.

consul/datadog_checks/consul/config_models/defaults.py

@@ -48,6 +48,10 @@ def instance_enable_legacy_tags_normalization():
     return True
 
 
+def instance_health_check_warning_events():
+    return False
+
+
 def instance_health_checks_cache_size():
     return 5000
 

consul/datadog_checks/consul/config_models/instance.py

@@ -76,6 +76,7 @@ class InstanceConfig(BaseModel):
     enable_legacy_tags_normalization: Optional[bool] = None
     extra_headers: Optional[MappingProxyType[str, Any]] = None
     headers: Optional[MappingProxyType[str, Any]] = None
+    health_check_warning_events: Optional[bool] = None
     health_checks_cache_size: Optional[int] = Field(None, ge=1)
     health_checks_cache_ttl: Optional[int] = Field(None, ge=1)
     kerberos_auth: Optional[Literal['required', 'optional', 'disabled']] = None

consul/datadog_checks/consul/consul.py

@@ -116,6 +116,12 @@ def __init__(self, name, init_config, instances):
         self.collect_health_checks = self.instance.get(
             'collect_health_checks', self.init_config.get('collect_health_checks', False)
         )
+        self.health_check_warning_events = is_affirmative(
+            self.instance.get(
+                'health_check_warning_events',
+                self.init_config.get('health_check_warning_events', False),
+            )
+        )
 
         if self.threads_count > 1:
             self.thread_pool = ThreadPool(self.threads_count)
@@ -407,22 +413,33 @@ def check(self, _):
                         self.gauge(HEALTH_CHECK_METRIC, status_value, tags=main_tags + node_tags)
                         self.health_checks[hc_id] = status_value
 
-                        if last_hc_value != status_value and status_value == 3:
-                            check_name = check.get("Name", "Consul Health Check")
-                            check_output = check.get("Output", "")
-                            self.event(
-                                {
-                                    "timestamp": timestamp(),
-                                    "event_type": "consul.check_failed",
-                                    "alert_type": "error",
-                                    "source_type_name": SOURCE_TYPE_NAME,
-                                    "msg_title": f"{check_name} Failed",
-                                    "aggregation_key": "consul.status_check",
-                                    "msg_text": f"Check {check_id} for service {service_name}, id: {service_id}"
-                                    f"failed on node {node_name}: {check_output}",
-                                    "tags": node_tags,
-                                }
-                            )
+                        if last_hc_value != status_value:
+                            if status_value == 3 or (status_value == 2 and self.health_check_warning_events):
+                                check_name = check.get("Name", "Consul Health Check")
+                                check_output = check.get("Output", "")
+
+                                if status_value == 3:
+                                    event_type = "consul.check_failed"
+                                    alert_type = "error"
+                                    label = "failed"
+                                else:
+                                    event_type = "consul.check_warning"
+                                    alert_type = "warning"
+                                    label = "warning"
+
+                                self.event(
+                                    {
+                                        "timestamp": timestamp(),
+                                        "event_type": event_type,
+                                        "alert_type": alert_type,
+                                        "source_type_name": SOURCE_TYPE_NAME,
+                                        "msg_title": f"{check_name} {label.capitalize()}",
+                                        "aggregation_key": "consul.status_check",
+                                        "msg_text": f"Check {check_id} for service {service_name}, id: {service_id}"
+                                        f"{label} on node {node_name}: {check_output}",
+                                        "tags": node_tags,
+                                    }
+                                )
 
                     if sc_id not in service_checks:
                         service_checks[sc_id] = {'status': status, 'tags': tags}

consul/datadog_checks/consul/data/conf.yaml.example

@@ -160,6 +160,12 @@ instances:
     #
     # health_checks_cache_ttl: 3600
 
+    ## @param health_check_warning_events - boolean - optional - default: false
+    ## Whether to emit an event when a Consul health check transitions to `warning`.
+    ## Events for critical health checks are always emitted when `collect_health_checks` is enabled.
+    #
+    # health_check_warning_events: true
+
     ## @param proxy - mapping - optional
     ## This overrides the `proxy` setting in `init_config`.
     ##

consul/tests/consul_mocks.py

@@ -306,6 +306,13 @@ def mock_get_coord_nodes_benchmark(num_nodes):
     return nodes
 
 
+def mock_get_health_check_with_warning(_):
+    checks = mock_get_health_check(_)
+    checks[0]["Status"] = "warning"
+    checks[0]["Output"] = "disk usage high"
+    return checks
+
+
 def mock_get_health_check(_):
     return [
         {

consul/tests/test_unit.py

@@ -331,6 +331,39 @@ def test_health_checks(aggregator, collect_health_checks, expected_metric_count,
     aggregator.assert_event(exact_match=False, count=expected_metric_count, **event)
 
 
+@pytest.mark.parametrize(
+    'health_check_warning_events, expected_warning_events',
+    [
+        pytest.param(True, 1, id="warning events enabled"),
+        pytest.param(False, 0, id="warning events disabled"),
+    ],
+)
+def test_health_check_warning_events(aggregator, health_check_warning_events, expected_warning_events):
+    config = consul_mocks.MOCK_CONFIG_DISABLE_SERVICE_TAG.copy()
+    config['collect_health_checks'] = True
+    config['health_check_warning_events'] = health_check_warning_events
+    consul_check = ConsulCheck(common.CHECK_NAME, {}, [config])
+    my_mocks = consul_mocks._get_consul_mocks()
+    my_mocks['consul_request'] = consul_mocks.mock_get_health_check_with_warning
+    consul_mocks.mock_check(consul_check, my_mocks)
+    consul_check.check(None)
+
+    warning_events = [e for e in aggregator.events if e['event_type'] == 'consul.check_warning']
+    assert len(warning_events) == expected_warning_events
+    if expected_warning_events:
+        assert warning_events[0]['alert_type'] == 'warning'
+        assert warning_events[0]['msg_title'] == "Service 'server-loadbalancer' check Warning"
+        assert (
+            warning_events[0]['msg_text']
+            == "Check server-loadbalancer for service server-loadbalancer, id: server-loadbalancerwarning "
+            "on node node-2: disk usage high"
+        )
+
+        consul_check.check(None)
+        warning_events = [e for e in aggregator.events if e['event_type'] == 'consul.check_warning']
+        assert len(warning_events) == 1
+
+
 def test_service_checks_disable_service_tag(aggregator):
     consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG_DISABLE_SERVICE_TAG])
     my_mocks = consul_mocks._get_consul_mocks()