feat(integrations): Send custom webhook_headers on SentryApp webhooks (#117090)

Attach a custom integration's `webhook_headers` to every outgoing
webhook
request. For errored requests, we always mask custom header values, but
display the headers that were sent for debuggability.

- Add `webhook_headers` to RpcSentryApp so delivery works across the
silo
  boundary (the install's sentry_app may be an RPC model).
- AppPlatformEvent merges custom headers under Sentry's own headers, so
Sentry's
headers always win and a custom Sentry-Hook-Signature can never spoof
the
  signed payload.
- The webhook request buffer (surfaced in the debug UI) now logs only
Sentry's
own headers via a new sentry_headers property; custom headers may carry
secrets and must never be persisted to Redis or shown there. The
outgoing
request still uses the merged headers, and safe_urlopen continues to
guard
  against SSRF to internal addresses.

Co-Authored-By: Claude <noreply@anthropic.com>

---

<sub>Stack created with <a
href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a
href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: 2 people

src/sentry/sentry_apps/api/serializers/app_platform_event.py

@@ -1,9 +1,11 @@
 from collections.abc import Mapping
 from enum import StrEnum
+from functools import cached_property
 from time import time
 from typing import Any, TypedDict
 from uuid import uuid4
 
+from sentry.sentry_apps.models.sentry_app import MASKED_VALUE
 from sentry.sentry_apps.models.sentry_app_installation import SentryAppInstallation
 from sentry.sentry_apps.services.app.model import RpcSentryAppInstallation
 from sentry.sentry_apps.utils.webhooks import SentryAppActionType, SentryAppResourceType
@@ -89,8 +91,13 @@ def body(self) -> str:
             )
         )
 
-    @property
-    def headers(self) -> dict[str, str]:
+    @cached_property
+    def sentry_headers(self) -> dict[str, str]:
+        """Headers Sentry sets on every webhook request.
+
+        Cached so the Request-ID, timestamp, and signature are computed once and
+        stay consistent between the sent request and the logged buffer entry.
+        """
         request_uuid = uuid4().hex
 
         return {
@@ -100,3 +107,39 @@ def headers(self) -> dict[str, str]:
             "Sentry-Hook-Timestamp": str(int(time())),
             "Sentry-Hook-Signature": self.install.sentry_app.build_signature(self.body),
         }
+
+    @property
+    def custom_headers(self) -> dict[str, str]:
+        """User-configured headers parsed from the SentryApp's webhook_headers."""
+        headers: dict[str, str] = {}
+        for header in self.install.sentry_app.webhook_headers or []:
+            name, separator, value = header.partition(":")
+            if separator:
+                headers[name.strip()] = value.strip()
+        return headers
+
+    @property
+    def headers(self) -> dict[str, str]:
+        # Sentry's headers are merged last so they always win: a custom header
+        # can never override the signature and spoof payload integrity.
+        return {**self.custom_headers, **self.sentry_headers}
+
+    @property
+    def masked_custom_headers(self) -> dict[str, str]:
+        """Custom header names with their values replaced by MASKED_VALUE.
+
+        Custom header values may carry secrets (e.g. bearer tokens), so they are
+        never persisted to the request buffer. The names are kept so the debug UI
+        can show which custom headers were sent without leaking the values.
+        """
+        return {name: MASKED_VALUE for name in self.custom_headers}
+
+    @property
+    def loggable_headers(self) -> dict[str, str]:
+        """Headers safe to record in the request buffer / debug UI.
+
+        Sentry's own headers in the clear, plus custom headers with masked values.
+        Sentry's headers are merged last so the buffer mirrors the precedence of
+        what was actually sent (see ``headers``).
+        """
+        return {**self.masked_custom_headers, **self.sentry_headers}

src/sentry/sentry_apps/services/app/model.py

@@ -85,6 +85,7 @@ class RpcSentryApp(RpcModel):
     uuid: str = ""
     events: list[str] = Field(default_factory=list)
     webhook_url: str | None = None
+    webhook_headers: list[str] = Field(repr=False, default_factory=list)
     is_alertable: bool = False
     is_disabled: bool = False
     is_published: bool = False
@@ -125,7 +126,7 @@ class RpcSentryAppInstallation(RpcModel):
     sentry_app: RpcSentryApp = Field(default_factory=lambda: RpcSentryApp())
     date_deleted: datetime.datetime | None = None
     uuid: str = ""
-    api_token: str | None = None
+    api_token: str | None = Field(repr=False, default=None)
 
 
 class RpcSentryAppComponent(RpcModel):

src/sentry/sentry_apps/services/app/serial.py

@@ -41,6 +41,7 @@ def serialize_sentry_app(
         uuid=app.uuid,
         events=app.events,
         webhook_url=app.webhook_url,
+        webhook_headers=app.webhook_headers,
         is_alertable=app.is_alertable,
         is_disabled=app.is_disabled,
         is_published=app.status == SentryAppStatus.PUBLISHED,

src/sentry/utils/sentry_apps/webhooks.py

@@ -13,7 +13,7 @@
 from requests.exceptions import ChunkedEncodingError, ConnectionError, Timeout
 from rest_framework import status
 
-from sentry import options
+from sentry import features, options
 from sentry.exceptions import RestrictedIPAddress
 from sentry.http import safe_urlopen
 from sentry.integrations.utils.metrics import EventLifecycle
@@ -195,6 +195,7 @@ def _circuit_breaker_allows_request(
 def _send_webhook_request(
     url: str,
     app_platform_event: AppPlatformEvent[T],
+    use_custom_headers: bool = False,
 ) -> Response:
     # We don't want to use the alarm in CONTROL silo as it's only used for installation webhooks which are v. low volume
     # Also that we aren't guaranteed to be in main thread
@@ -212,7 +213,9 @@ def _send_webhook_request(
         return safe_urlopen(
             url=url,
             data=app_platform_event.body,
-            headers=app_platform_event.headers,
+            headers=app_platform_event.headers
+            if use_custom_headers
+            else app_platform_event.sentry_headers,
             timeout=options.get("sentry-apps.webhook.timeout.sec"),
         )
 
@@ -262,19 +265,26 @@ def send_and_save_webhook_request(
         )
 
         assert url is not None
+        custom_headers_enabled = False
         try:
             owner_context = organization_service.get_organization_by_id(
                 id=sentry_app.owner_id,
                 include_projects=False,
                 include_teams=False,
             )
             owner_org = owner_context.organization if owner_context is not None else None
+            if owner_org is not None:
+                custom_headers_enabled = features.has(
+                    "organizations:sentry-apps-custom-webhook-headers", owner_org
+                )
             circuit_breaker = _create_circuit_breaker(sentry_app)
             if not _circuit_breaker_allows_request(circuit_breaker, sentry_app, lifecycle):
                 return Response()
 
             with circuit_breaker_tracking(circuit_breaker):
-                response = _send_webhook_request(url, app_platform_event)
+                response = _send_webhook_request(
+                    url, app_platform_event, use_custom_headers=custom_headers_enabled
+                )
 
         except WebhookTimeoutError:
             if circuit_breaker and circuit_breaker.is_open() and owner_org is not None:
@@ -307,7 +317,9 @@ def send_and_save_webhook_request(
                 org_id=org_id,
                 event=event,
                 url=url,
-                headers=app_platform_event.headers,
+                headers=app_platform_event.loggable_headers
+                if custom_headers_enabled
+                else app_platform_event.sentry_headers,
             )
             lifecycle.record_halt(e)
             # Re-raise the exception because some of these tasks might retry on the exception
@@ -338,7 +350,9 @@ def send_and_save_webhook_request(
             error_id=response.headers.get("Sentry-Hook-Error"),
             project_id=project_id,
             response=response,
-            headers=app_platform_event.headers,
+            headers=app_platform_event.loggable_headers
+            if custom_headers_enabled
+            else app_platform_event.sentry_headers,
         )
 
         debug_logging_enabled = (

tests/sentry/sentry_apps/api/serializers/test_app_platform_event.py

@@ -4,6 +4,7 @@
 import orjson
 
 from sentry.sentry_apps.api.serializers.app_platform_event import AppPlatformEvent
+from sentry.sentry_apps.models.sentry_app import MASKED_VALUE, SentryApp
 from sentry.sentry_apps.utils.webhooks import (
     InstallationActionType,
     IssueActionType,
@@ -91,3 +92,88 @@ def test_user_actor(self) -> None:
         assert result.headers["Content-Type"] == "application/json"
         assert result.headers["Sentry-Hook-Resource"] == "installation"
         assert result.headers["Sentry-Hook-Signature"] == signature
+
+    def _event_for_app_with_headers(
+        self, headers: list[str]
+    ) -> tuple[SentryApp, AppPlatformEvent[dict[str, Any]]]:
+        sentry_app = self.create_sentry_app(organization=self.organization, webhook_headers=headers)
+        install = self.create_sentry_app_installation(
+            organization=self.organization, slug=sentry_app.slug
+        )
+        event = AppPlatformEvent[dict[str, Any]](
+            resource=SentryAppResourceType.ISSUE,
+            action=IssueActionType.ASSIGNED,
+            install=install,
+            data={},
+        )
+        return sentry_app, event
+
+    def test_custom_webhook_headers_are_merged(self) -> None:
+        _, result = self._event_for_app_with_headers(
+            ["X-Custom: value", "Authorization: Bearer token"]
+        )
+
+        assert result.headers["X-Custom"] == "value"
+        assert result.headers["Authorization"] == "Bearer token"
+        # Sentry's own headers are still present.
+        assert result.headers["Content-Type"] == "application/json"
+
+    def test_custom_headers_cannot_override_sentry_headers(self) -> None:
+        sentry_app, result = self._event_for_app_with_headers(
+            ["Sentry-Hook-Signature: spoofed", "Content-Type: text/plain"]
+        )
+
+        # Sentry's headers win the merge so the signature can't be spoofed.
+        assert result.headers["Sentry-Hook-Signature"] == sentry_app.build_signature(result.body)
+        assert result.headers["Content-Type"] == "application/json"
+
+    def test_sentry_headers_exclude_custom_headers(self) -> None:
+        # sentry_headers carries only Sentry's own headers in the clear; custom
+        # headers (which may contain secrets) must never appear there unmasked.
+        _, result = self._event_for_app_with_headers(["Authorization: Bearer secret"])
+
+        assert "Authorization" not in result.sentry_headers
+        assert result.headers["Authorization"] == "Bearer secret"
+
+    def test_loggable_headers_mask_custom_header_values(self) -> None:
+        # loggable_headers is what gets recorded in the request buffer / debug UI:
+        # custom header names are kept so users can confirm they were sent, but the
+        # values are masked so secrets never get persisted.
+        _, result = self._event_for_app_with_headers(
+            ["X-Custom: value", "Authorization: Bearer secret"]
+        )
+
+        loggable = result.loggable_headers
+        # Names are visible so the debug UI shows which custom headers were sent.
+        assert loggable["X-Custom"] == MASKED_VALUE
+        assert loggable["Authorization"] == MASKED_VALUE
+        # The real secret value is never recorded in the buffer.
+        assert "Bearer secret" not in loggable.values()
+        # Sentry's own headers stay in the clear so they remain useful for debugging.
+        assert loggable["Content-Type"] == "application/json"
+        assert loggable["Sentry-Hook-Signature"] == result.sentry_headers["Sentry-Hook-Signature"]
+
+    def test_loggable_headers_sentry_headers_win_collisions(self) -> None:
+        # A custom header that collides with a Sentry header must not shadow the real
+        # value in the buffer, mirroring the precedence of what was actually sent.
+        _, result = self._event_for_app_with_headers(["Content-Type: text/plain"])
+
+        assert result.loggable_headers["Content-Type"] == "application/json"
+
+    def test_sentry_headers_are_stable_across_calls(self) -> None:
+        # The Request-ID/timestamp logged to the buffer must match what was sent,
+        # so sentry_headers is computed once per event rather than per access.
+        result = AppPlatformEvent[dict[str, Any]](
+            resource=SentryAppResourceType.ISSUE,
+            action=IssueActionType.ASSIGNED,
+            install=self.install,
+            data={},
+        )
+
+        first = result.sentry_headers
+        second = result.sentry_headers
+        assert first["Request-ID"] == second["Request-ID"]
+        assert first["Sentry-Hook-Timestamp"] == second["Sentry-Hook-Timestamp"]
+        # The headers actually sent carry the same values that get logged.
+        assert result.headers["Request-ID"] == first["Request-ID"]
+        assert result.headers["Sentry-Hook-Timestamp"] == first["Sentry-Hook-Timestamp"]

tests/sentry/utils/sentry_apps/test_webhooks.py

@@ -8,15 +8,17 @@
 
 from sentry.notifications.platform.service import NotificationService
 from sentry.sentry_apps.api.serializers.app_platform_event import AppPlatformEvent
+from sentry.sentry_apps.models.sentry_app import MASKED_VALUE
 from sentry.sentry_apps.utils.webhooks import IssueActionType, SentryAppResourceType
-from sentry.shared_integrations.exceptions import ApiHostError
+from sentry.shared_integrations.exceptions import ApiHostError, ClientError
 from sentry.testutils.asserts import assert_failure_metric
 from sentry.testutils.cases import TestCase
 from sentry.testutils.helpers.features import with_feature
 from sentry.testutils.helpers.options import override_options
 from sentry.testutils.silo import cell_silo_test
 from sentry.utils import redis
 from sentry.utils.circuit_breaker2 import CircuitBreaker
+from sentry.utils.sentry_apps import SentryAppWebhookRequestsBuffer
 from sentry.utils.sentry_apps.webhooks import WebhookTimeoutError, send_and_save_webhook_request
 
 
@@ -145,6 +147,85 @@ def test_http_error_response_records_success_and_raises(
 
         mock_record_success.assert_called_once()
 
+    @with_feature("organizations:sentry-apps-custom-webhook-headers")
+    @override_options(CIRCUIT_BREAKER_OPTIONS)
+    @patch("sentry.utils.sentry_apps.webhooks.safe_urlopen")
+    def test_error_response_buffers_masked_custom_headers(self, mock_safe_urlopen):
+        """A failed delivery records masked custom headers in the request buffer, so the
+        debug UI shows which custom headers were sent without persisting their secrets."""
+        sentry_app = self.create_sentry_app(
+            name="HeaderApp",
+            organization=self.organization,
+            webhook_url="https://example.com/webhook",
+            published=True,
+            webhook_headers=["Authorization: Bearer super-secret"],
+        )
+        install = self.create_sentry_app_installation(
+            organization=self.organization, slug=sentry_app.slug
+        )
+        event = AppPlatformEvent(
+            resource=SentryAppResourceType.ISSUE,
+            action=IssueActionType.CREATED,
+            install=install,
+            data={"test": "data"},
+        )
+        mock_safe_urlopen.return_value = _MockResponse(
+            {}, "{}", "", False, 401, _raise_status_false, None
+        )
+
+        with pytest.raises(ClientError):
+            send_and_save_webhook_request(sentry_app, event)
+
+        requests = SentryAppWebhookRequestsBuffer(sentry_app).get_requests(errors_only=True)
+        assert len(requests) == 1
+        headers = requests[0].get("request_headers")
+        assert headers is not None
+        # The custom header name is recorded but its value is masked.
+        assert headers["Authorization"] == MASKED_VALUE
+        assert "Bearer super-secret" not in headers.values()
+        # Sentry's own headers are still recorded in the clear.
+        assert headers["Content-Type"] == "application/json"
+
+    @override_options(CIRCUIT_BREAKER_OPTIONS)
+    @patch("sentry.utils.sentry_apps.webhooks.safe_urlopen")
+    def test_custom_headers_not_sent_or_logged_without_flag(self, mock_safe_urlopen):
+        """Without the feature flag, custom headers are stripped from both the request
+        and the buffer log."""
+        sentry_app = self.create_sentry_app(
+            name="HeaderApp",
+            organization=self.organization,
+            webhook_url="https://example.com/webhook",
+            published=True,
+            webhook_headers=["Authorization: Bearer super-secret"],
+        )
+        install = self.create_sentry_app_installation(
+            organization=self.organization, slug=sentry_app.slug
+        )
+        event = AppPlatformEvent(
+            resource=SentryAppResourceType.ISSUE,
+            action=IssueActionType.CREATED,
+            install=install,
+            data={"test": "data"},
+        )
+        mock_safe_urlopen.return_value = _MockResponse(
+            {}, "{}", "", False, 401, _raise_status_false, None
+        )
+
+        with pytest.raises(ClientError):
+            send_and_save_webhook_request(sentry_app, event)
+
+        # Custom header must not appear in the outbound request.
+        call_headers = mock_safe_urlopen.call_args.kwargs["headers"]
+        assert "Authorization" not in call_headers
+
+        # Custom header must not appear in the buffer log either.
+        requests = SentryAppWebhookRequestsBuffer(sentry_app).get_requests(errors_only=True)
+        assert len(requests) == 1
+        headers = requests[0].get("request_headers")
+        assert headers is not None
+        assert "Authorization" not in headers
+        assert headers["Content-Type"] == "application/json"
+
 
 @cell_silo_test
 class WebhookCircuitBreakerNotifyTest(TestCase):