add observability to checkmate test (#854)

* add observability to checkmate test

* add observability to checkmate test

* update format and lint

Author: santicomp2014

tests/unit/viahtml/views/status_test.py

@@ -1,3 +1,4 @@
+import logging
 from unittest.mock import create_autospec
 
 import pytest
@@ -82,6 +83,84 @@ def test_it_sends_test_messages_to_sentry(self, context, view, capture_message):
             "Test message from Via HTML's status view"
         )
 
+    def test_it_logs_checkmate_failure_details(self, context, view, checkmate, caplog):
+        exc = CheckmateException("Connection timed out")
+        checkmate.check_url.side_effect = exc
+        context.query_params = {"include-checkmate": [""]}
+
+        with caplog.at_level(logging.ERROR, logger="viahtml.views.status"):
+            view(context)
+
+        assert any(
+            "Checkmate status check failed" in record.message
+            and "Connection timed out" in record.message
+            for record in caplog.records
+        ), f"Expected checkmate failure log, got: {[r.message for r in caplog.records]}"
+
+    def test_it_logs_checkmate_success(self, context, view, caplog):
+        context.query_params = {"include-checkmate": [""]}
+
+        with caplog.at_level(logging.INFO, logger="viahtml.views.status"):
+            view(context)
+
+        assert any(
+            "Checkmate status check succeeded" in record.message
+            for record in caplog.records
+        ), f"Expected checkmate success log, got: {[r.message for r in caplog.records]}"
+
+    def test_it_creates_sentry_span_on_checkmate_failure(
+        self, context, view, checkmate, sentry_start_span, sentry_capture_exception
+    ):  # pylint:disable=too-many-arguments
+        exc = CheckmateException("Connection timed out")
+        checkmate.check_url.side_effect = exc
+        context.query_params = {"include-checkmate": [""]}
+
+        view(context)
+
+        sentry_start_span.assert_called_once_with(
+            op="checkmate.status_check",
+            name="Check Checkmate health via check_url",
+        )
+        span = sentry_start_span.return_value.__enter__.return_value
+        span.set_status.assert_called_once_with("internal_error")
+        span.set_data.assert_any_call("error.type", "CheckmateException")
+        span.set_data.assert_any_call("error.message", "Connection timed out")
+        sentry_capture_exception.assert_called_once_with(exc)
+
+    def test_it_creates_sentry_span_on_checkmate_success(
+        self, context, view, sentry_start_span, sentry_capture_exception
+    ):
+        context.query_params = {"include-checkmate": [""]}
+
+        view(context)
+
+        sentry_start_span.assert_called_once_with(
+            op="checkmate.status_check",
+            name="Check Checkmate health via check_url",
+        )
+        span = sentry_start_span.return_value.__enter__.return_value
+        span.set_status.assert_called_once_with("ok")
+        sentry_capture_exception.assert_not_called()
+
+    def test_it_adds_sentry_breadcrumb_on_checkmate_failure(
+        self, context, view, checkmate, sentry_add_breadcrumb
+    ):
+        exc = CheckmateException("Connection timed out")
+        checkmate.check_url.side_effect = exc
+        context.query_params = {"include-checkmate": [""]}
+
+        view(context)
+
+        sentry_add_breadcrumb.assert_called_once_with(
+            category="checkmate",
+            message="Checkmate status check failed: Connection timed out",
+            level="error",
+            data={
+                "exception_type": "CheckmateException",
+                "exception_message": "Connection timed out",
+            },
+        )
+
     @pytest.fixture
     def checkmate(self):
         return create_autospec(CheckmateClient, instance=True, spec_set=True)
@@ -95,6 +174,18 @@ def context(self, context):
     def view(self, checkmate):
         return StatusView(checkmate)
 
+    @pytest.fixture(autouse=True)
+    def sentry_start_span(self, patch):
+        return patch("viahtml.views.status.sentry_sdk.start_span")
+
+    @pytest.fixture(autouse=True)
+    def sentry_capture_exception(self, patch):
+        return patch("viahtml.views.status.sentry_sdk.capture_exception")
+
+    @pytest.fixture(autouse=True)
+    def sentry_add_breadcrumb(self, patch):
+        return patch("viahtml.views.status.sentry_sdk.add_breadcrumb")
+
 
 @pytest.fixture(autouse=True)
 def capture_message(patch):

viahtml/views/status.py

@@ -1,8 +1,13 @@
+import logging
+import traceback
 from http import HTTPStatus
 
+import sentry_sdk
 from checkmatelib import CheckmateException
 from sentry_sdk import capture_message
 
+LOG = logging.getLogger(__name__)
+
 
 class StatusView:
     def __init__(self, checkmate):
@@ -22,12 +27,7 @@ def __call__(self, context):
         http_status = HTTPStatus.OK
 
         if "include-checkmate" in context.query_params:
-            try:
-                self._checkmate.check_url("https://example.com/")
-            except CheckmateException:
-                body["down"] = ["checkmate"]
-            else:
-                body["okay"] = ["checkmate"]
+            self._check_checkmate(body)
 
         # If any of the components checked above were down then report the
         # status check as a whole as being down.
@@ -44,3 +44,37 @@ def __call__(self, context):
             http_status=http_status,
             headers={"Cache-Control": "max-age=0, must-revalidate, no-cache, no-store"},
         )
+
+    def _check_checkmate(self, body):
+        with sentry_sdk.start_span(
+            op="checkmate.status_check",
+            name="Check Checkmate health via check_url",
+        ) as span:
+            LOG.info("Checking checkmate status via check_url")
+            try:
+                self._checkmate.check_url("https://example.com/")
+            except CheckmateException as exc:
+                LOG.error(
+                    "Checkmate status check failed: %s: %s\n%s",
+                    type(exc).__name__,
+                    exc,
+                    traceback.format_exc(),
+                )
+                span.set_status("internal_error")
+                span.set_data("error.type", type(exc).__name__)
+                span.set_data("error.message", str(exc))
+                sentry_sdk.add_breadcrumb(
+                    category="checkmate",
+                    message=f"Checkmate status check failed: {exc}",
+                    level="error",
+                    data={
+                        "exception_type": type(exc).__name__,
+                        "exception_message": str(exc),
+                    },
+                )
+                sentry_sdk.capture_exception(exc)
+                body["down"] = ["checkmate"]
+            else:
+                LOG.info("Checkmate status check succeeded")
+                span.set_status("ok")
+                body["okay"] = ["checkmate"]