feat(sentry): add optional Sentry error tracking integration

major · major · commit 0c9ae903cf41 · 2026-04-24T10:27:48.000-05:00
Add Sentry error tracking that activates only when a SENTRY_DSN
environment variable is set. Includes custom trace sampling to exclude
health check, metrics, and root routes from tracing.

- Add sentry-sdk[fastapi] runtime dependency
- Add Sentry configuration constants to constants.py
- Create src/sentry.py with initialize_sentry() and trace sampler
- Wire Sentry init into FastAPI lifespan (startup + shutdown flush)
- Add comprehensive unit tests covering all branches

Signed-off-by: Major Hayden &lt;major@redhat.com&gt;
diff --git a/pyproject.toml b/pyproject.toml
@@ -73,6 +73,8 @@ dependencies = [
     # To be able to fix multiple CVEs, also LCORE-1117
     "requests>=2.33.0",
     "datasets>=4.7.0",
+    # Used for error tracking and monitoring
+    "sentry-sdk[fastapi]>=2.58.0",
 ]
 
 
diff --git a/src/app/main.py b/src/app/main.py
@@ -4,6 +4,7 @@
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 
+import sentry_sdk  # pyright: ignore[reportMissingImports]
 from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
@@ -22,6 +23,7 @@
 from configuration import configuration
 from log import get_logger
 from models.responses import InternalServerErrorResponse
+from sentry import initialize_sentry
 from utils.common import register_mcp_servers_async
 from utils.llama_stack_version import check_llama_stack_version
 
@@ -44,6 +46,8 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
     """
     configuration.load_configuration(os.environ["LIGHTSPEED_STACK_CONFIG_PATH"])
 
+    initialize_sentry()
+
     azure_config = configuration.configuration.azure_entra_id
     if azure_config is not None:
         AzureEntraIDManager().set_config(azure_config)
@@ -81,8 +85,13 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
     yield
 
     # Cleanup resources on shutdown
-    await shutdown_background_topic_summary_tasks()
-    await A2AStorageFactory.cleanup()
+    try:
+        await shutdown_background_topic_summary_tasks()
+        await A2AStorageFactory.cleanup()
+    finally:
+        # Flush pending Sentry events after cleanup so any errors during
+        # shutdown are captured before the process exits.
+        sentry_sdk.flush(timeout=2)
     logger.info("App shutdown complete")
 
 
diff --git a/src/constants.py b/src/constants.py
@@ -249,3 +249,27 @@
 RLSAPI_V1_QUESTION_MAX_LENGTH: Final[int] = 32_768
 # Maximum character length for the serialized /v1/responses request body (64 KiB)
 RESPONSES_REQUEST_MAX_SIZE: Final[int] = 65_536
+
+# Sentry configuration constants
+# Environment variable name for the Sentry DSN (Data Source Name)
+SENTRY_DSN_ENV_VAR: Final[str] = "SENTRY_DSN"
+# Environment variable name for the Sentry environment tag
+SENTRY_ENVIRONMENT_ENV_VAR: Final[str] = "SENTRY_ENVIRONMENT"
+# Default Sentry environment when SENTRY_ENVIRONMENT is not set
+SENTRY_DEFAULT_ENVIRONMENT: Final[str] = "development"
+# Default trace sample rate (fraction of transactions to capture)
+SENTRY_DEFAULT_TRACES_SAMPLE_RATE: Final[float] = 0.25
+# Routes excluded from Sentry trace sampling (health checks, metrics, root).
+# Note: health and metrics routers are mounted WITHOUT a /v1 prefix
+# (see the setup_routers function in src/app/routers.py), so ASGI paths are
+# /readiness, /liveness, /metrics.
+SENTRY_EXCLUDED_ROUTES: Final[tuple[str, ...]] = (
+    "/readiness",
+    "/liveness",
+    "/metrics",
+    "/",
+)
+# Environment variable name for the Sentry CA certificate bundle path.
+# Set this to a file path (e.g. /etc/pki/tls/certs/ca-bundle.crt) when
+# connecting to a Sentry instance that uses a private or internal CA.
+SENTRY_CA_CERTS_ENV_VAR: Final[str] = "SENTRY_CA_CERTS"
diff --git a/src/sentry.py b/src/sentry.py
@@ -0,0 +1,112 @@
+"""Sentry error tracking initialization and configuration."""
+
+import os
+
+import sentry_sdk  # pyright: ignore[reportMissingImports]
+from sentry_sdk.integrations.fastapi import (  # pyright: ignore[reportMissingImports]
+    FastApiIntegration,
+)
+
+import version
+from constants import (
+    SENTRY_CA_CERTS_ENV_VAR,
+    SENTRY_DEFAULT_ENVIRONMENT,
+    SENTRY_DEFAULT_TRACES_SAMPLE_RATE,
+    SENTRY_DSN_ENV_VAR,
+    SENTRY_ENVIRONMENT_ENV_VAR,
+    SENTRY_EXCLUDED_ROUTES,
+)
+from log import get_logger
+
+logger = get_logger(__name__)
+
+
+def sentry_traces_sampler(tracing_context: dict) -> float:
+    """
+    Determine the trace sample rate for a given request.
+
+    Excludes health check, metrics, and root routes from trace sampling to
+    reduce noise. All other routes use the default sample rate.
+
+    Parameters:
+    ----------
+        tracing_context (dict): The Sentry tracing context containing ASGI
+            scope information, including the request path.
+
+    Returns:
+    -------
+        float: 0.0 for excluded routes (no sampling), or
+            SENTRY_DEFAULT_TRACES_SAMPLE_RATE for all other routes.
+    """
+    asgi_scope = tracing_context.get("asgi_scope", {})
+    path = asgi_scope.get("path") if isinstance(asgi_scope, dict) else None
+
+    if path is not None:
+        if path == "/":
+            return 0.0
+        if any(
+            route != "/" and path.endswith(route) for route in SENTRY_EXCLUDED_ROUTES
+        ):
+            return 0.0
+
+    return SENTRY_DEFAULT_TRACES_SAMPLE_RATE
+
+
+def initialize_sentry() -> None:
+    """
+    Initialize Sentry error tracking if a DSN is configured.
+
+    Reads the SENTRY_DSN environment variable. If not set or empty, logs an
+    informational message and returns without initializing Sentry. When a DSN
+    is present, initializes the Sentry SDK with custom trace sampling, FastAPI
+    integration, and optional CA certificate configuration.
+
+    When SENTRY_CA_CERTS is set to a file path, that certificate bundle is
+    passed to the SDK for Sentry instances using private or internal CAs.
+
+    The DSN value is never logged to prevent accidental credential exposure.
+
+    Parameters:
+    ----------
+        None
+
+    Returns:
+    -------
+        None
+    """
+    dsn = os.environ.get(SENTRY_DSN_ENV_VAR)
+
+    if not dsn:
+        logger.info("Sentry DSN not configured, skipping initialization")
+        return
+
+    ca_certs = None
+    ca_certs_path = os.environ.get(SENTRY_CA_CERTS_ENV_VAR)
+    if ca_certs_path:
+        if os.path.exists(ca_certs_path):
+            ca_certs = ca_certs_path
+        else:
+            logger.warning(
+                "CA cert file specified by %s not found at %s; "
+                "proceeding without custom CA certs",
+                SENTRY_CA_CERTS_ENV_VAR,
+                ca_certs_path,
+            )
+
+    environment = os.environ.get(SENTRY_ENVIRONMENT_ENV_VAR, SENTRY_DEFAULT_ENVIRONMENT)
+
+    try:
+        sentry_sdk.init(
+            dsn=dsn,
+            environment=environment,
+            traces_sampler=sentry_traces_sampler,
+            send_default_pii=False,
+            ca_certs=ca_certs,
+            integrations=[FastApiIntegration(http_methods_to_capture=("POST",))],
+            release=f"lightspeed-stack@{version.__version__}",
+        )
+        logger.info("Sentry initialized")
+    except Exception:  # pylint: disable=broad-exception-caught
+        logger.exception(
+            "Failed to initialize Sentry, continuing without error tracking"
+        )
diff --git a/tests/unit/test_sentry.py b/tests/unit/test_sentry.py
@@ -0,0 +1,186 @@
+"""Unit tests for functions defined in src/sentry.py."""
+
+import pytest
+from pytest_mock import MockerFixture
+
+from constants import (
+    SENTRY_CA_CERTS_ENV_VAR,
+    SENTRY_DEFAULT_ENVIRONMENT,
+    SENTRY_DEFAULT_TRACES_SAMPLE_RATE,
+    SENTRY_DSN_ENV_VAR,
+    SENTRY_ENVIRONMENT_ENV_VAR,
+    SENTRY_EXCLUDED_ROUTES,
+)
+from sentry import initialize_sentry, sentry_traces_sampler
+
+
+class TestInitializeSentry:
+    """Tests for the initialize_sentry function."""
+
+    def test_dsn_not_set(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test that Sentry is not initialized when DSN env var is unset."""
+        monkeypatch.delenv(SENTRY_DSN_ENV_VAR, raising=False)
+        mock_init = mocker.patch("sentry.sentry_sdk.init")
+
+        initialize_sentry()
+
+        mock_init.assert_not_called()
+
+    def test_dsn_empty_string(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test that Sentry is not initialized when DSN is an empty string."""
+        monkeypatch.setenv(SENTRY_DSN_ENV_VAR, "")
+        mock_init = mocker.patch("sentry.sentry_sdk.init")
+
+        initialize_sentry()
+
+        mock_init.assert_not_called()
+
+    def test_dsn_set_no_ca_certs(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test Sentry init without CA certs env var uses ca_certs=None."""
+        dsn = "https://key@sentry.io/123"
+        monkeypatch.setenv(SENTRY_DSN_ENV_VAR, dsn)
+        monkeypatch.delenv(SENTRY_ENVIRONMENT_ENV_VAR, raising=False)
+        monkeypatch.delenv(SENTRY_CA_CERTS_ENV_VAR, raising=False)
+        mock_init = mocker.patch("sentry.sentry_sdk.init")
+
+        initialize_sentry()
+
+        mock_init.assert_called_once()
+        call_kwargs = mock_init.call_args.kwargs
+        assert call_kwargs["dsn"] == dsn
+        assert call_kwargs["ca_certs"] is None
+        assert call_kwargs["send_default_pii"] is False
+        assert call_kwargs["environment"] == SENTRY_DEFAULT_ENVIRONMENT
+        assert call_kwargs["release"].startswith("lightspeed-stack@")
+
+    def test_ca_certs_file_exists(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test that ca_certs is set when SENTRY_CA_CERTS points to an existing file."""
+        ca_path = "/etc/pki/tls/certs/ca-bundle.crt"
+        monkeypatch.setenv(SENTRY_DSN_ENV_VAR, "https://key@sentry.io/123")
+        monkeypatch.setenv(SENTRY_CA_CERTS_ENV_VAR, ca_path)
+        monkeypatch.delenv(SENTRY_ENVIRONMENT_ENV_VAR, raising=False)
+        mock_init = mocker.patch("sentry.sentry_sdk.init")
+        mocker.patch("sentry.os.path.exists", return_value=True)
+
+        initialize_sentry()
+
+        mock_init.assert_called_once()
+        assert mock_init.call_args.kwargs["ca_certs"] == ca_path
+
+    def test_ca_certs_file_missing(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test that ca_certs is None and a warning is logged when the cert file is missing."""
+        ca_path = "/nonexistent/ca-bundle.crt"
+        monkeypatch.setenv(SENTRY_DSN_ENV_VAR, "https://key@sentry.io/123")
+        monkeypatch.setenv(SENTRY_CA_CERTS_ENV_VAR, ca_path)
+        monkeypatch.delenv(SENTRY_ENVIRONMENT_ENV_VAR, raising=False)
+        mock_init = mocker.patch("sentry.sentry_sdk.init")
+        mocker.patch("sentry.os.path.exists", return_value=False)
+        mock_logger = mocker.patch("sentry.logger")
+
+        initialize_sentry()
+
+        mock_init.assert_called_once()
+        assert mock_init.call_args.kwargs["ca_certs"] is None
+        mock_logger.warning.assert_called_once_with(
+            "CA cert file specified by %s not found at %s; "
+            "proceeding without custom CA certs",
+            SENTRY_CA_CERTS_ENV_VAR,
+            ca_path,
+        )
+
+    def test_custom_environment(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test that a custom SENTRY_ENVIRONMENT value is passed to init."""
+        monkeypatch.setenv(SENTRY_DSN_ENV_VAR, "https://key@sentry.io/123")
+        monkeypatch.setenv(SENTRY_ENVIRONMENT_ENV_VAR, "staging")
+        mock_init = mocker.patch("sentry.sentry_sdk.init")
+
+        initialize_sentry()
+
+        mock_init.assert_called_once()
+        assert mock_init.call_args.kwargs["environment"] == "staging"
+
+    def test_default_environment(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test that default environment is used when env var is unset."""
+        monkeypatch.setenv(SENTRY_DSN_ENV_VAR, "https://key@sentry.io/123")
+        monkeypatch.delenv(SENTRY_ENVIRONMENT_ENV_VAR, raising=False)
+        mock_init = mocker.patch("sentry.sentry_sdk.init")
+
+        initialize_sentry()
+
+        mock_init.assert_called_once()
+        assert mock_init.call_args.kwargs["environment"] == SENTRY_DEFAULT_ENVIRONMENT
+
+    def test_init_failure_does_not_raise(
+        self, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture
+    ) -> None:
+        """Test that a failure during sentry_sdk.init does not propagate."""
+        monkeypatch.setenv(SENTRY_DSN_ENV_VAR, "https://key@sentry.io/123")
+        monkeypatch.delenv(SENTRY_ENVIRONMENT_ENV_VAR, raising=False)
+        monkeypatch.delenv(SENTRY_CA_CERTS_ENV_VAR, raising=False)
+        mocker.patch(
+            "sentry.sentry_sdk.init", side_effect=RuntimeError("connection failed")
+        )
+        mock_logger = mocker.patch("sentry.logger")
+
+        initialize_sentry()
+
+        mock_logger.exception.assert_called_once_with(
+            "Failed to initialize Sentry, continuing without error tracking"
+        )
+
+
+class TestSentryTracesSampler:
+    """Tests for the sentry_traces_sampler function."""
+
+    @pytest.mark.parametrize(
+        "path",
+        list(SENTRY_EXCLUDED_ROUTES),
+        ids=[r.lstrip("/") or "root" for r in SENTRY_EXCLUDED_ROUTES],
+    )
+    def test_excluded_routes_return_zero(self, path: str) -> None:
+        """Test that excluded routes produce a sample rate of 0.0."""
+        context: dict = {"asgi_scope": {"path": path}}
+        assert sentry_traces_sampler(context) == 0.0
+
+    def test_excluded_route_suffix_match(self) -> None:
+        """Test that suffix matching works for excluded routes (e.g. /prometheus/metrics)."""
+        context: dict = {"asgi_scope": {"path": "/prometheus/metrics"}}
+        assert sentry_traces_sampler(context) == 0.0
+
+    @pytest.mark.parametrize(
+        "path",
+        ["/v1/query", "/v1/feedback", "/v1/query/"],
+        ids=["query", "feedback", "query_trailing_slash"],
+    )
+    def test_normal_routes_return_default_rate(self, path: str) -> None:
+        """Test that non-excluded routes use the default sample rate."""
+        context: dict = {"asgi_scope": {"path": path}}
+        assert sentry_traces_sampler(context) == SENTRY_DEFAULT_TRACES_SAMPLE_RATE
+
+    def test_empty_context(self) -> None:
+        """Test that an empty tracing context returns the default sample rate."""
+        assert sentry_traces_sampler({}) == SENTRY_DEFAULT_TRACES_SAMPLE_RATE
+
+    def test_missing_path_in_asgi_scope(self) -> None:
+        """Test that missing path key in asgi_scope returns the default rate."""
+        context: dict = {"asgi_scope": {}}
+        assert sentry_traces_sampler(context) == SENTRY_DEFAULT_TRACES_SAMPLE_RATE
+
+    def test_none_path_value(self) -> None:
+        """Test that a None path value returns the default sample rate."""
+        context: dict = {"asgi_scope": {"path": None}}
+        assert sentry_traces_sampler(context) == SENTRY_DEFAULT_TRACES_SAMPLE_RATE
diff --git a/uv.lock b/uv.lock

Original file line number	Diff line number	Diff line change
`@@ -73,6 +73,8 @@ dependencies = [`
`73`	`73`	`# To be able to fix multiple CVEs, also LCORE-1117`
`74`	`74`	`"requests>=2.33.0",`
`75`	`75`	`"datasets>=4.7.0",`
	`76`	`+ # Used for error tracking and monitoring`
	`77`	`+ "sentry-sdk[fastapi]>=2.58.0",`
`76`	`78`	`]`
`77`	`79`
`78`	`80`