Correlate Sentry errors with OTel traces (#1320)

* Correlate Sentry errors with OTel traces

* Default the Sentry OTel verification flags on in the e2e cloud stack

Author: RhysSullivan

apps/cloud/src/api/error-response.ts

@@ -1,7 +1,7 @@
 import { Cause, Data, Effect, Predicate, Result } from "effect";
 import { HttpServerRespondable, HttpServerResponse } from "effect/unstable/http";
 
-import { captureCause } from "../observability";
+import { captureCause, captureCauseEffect } from "../observability";
 
 // Implements `Respondable` so the framework's default cause→response
 // pipeline (`HttpServerRespondable.toResponseOrElse`) renders this as the
@@ -13,13 +13,14 @@ export class HttpResponseError extends Data.TaggedError("HttpResponseError")<{
   readonly message: string;
 }> {
   [HttpServerRespondable.symbol](): Effect.Effect<HttpServerResponse.HttpServerResponse> {
-    if (this.status >= 500) captureCause(this);
-    return Effect.succeed(
-      HttpServerResponse.jsonUnsafe(
-        { error: this.message, code: this.code },
-        { status: this.status },
-      ),
-    );
+    const self = this;
+    return Effect.gen(function* () {
+      if (self.status >= 500) yield* captureCauseEffect(self);
+      return HttpServerResponse.jsonUnsafe(
+        { error: self.message, code: self.code },
+        { status: self.status },
+      );
+    });
   }
 }
 
@@ -65,3 +66,15 @@ export const toErrorServerResponse = (error: unknown): HttpServerResponse.HttpSe
     { status: mapped.status },
   );
 };
+
+export const toErrorServerResponseEffect = (
+  error: unknown,
+): Effect.Effect<HttpServerResponse.HttpServerResponse> =>
+  Effect.gen(function* () {
+    const mapped = toHttpResponseError(error);
+    if (mapped.status >= 500) yield* captureCauseEffect(mapped);
+    return HttpServerResponse.jsonUnsafe(
+      { error: mapped.message, code: mapped.code },
+      { status: mapped.status },
+    );
+  });

apps/cloud/src/engine/execution-gate.ts

@@ -24,12 +24,12 @@
 // `CodeCompilationError` / `SandboxRuntimeError` into `ExecuteResult.error`.
 // ---------------------------------------------------------------------------
 
-import * as Sentry from "@sentry/cloudflare";
 import { Data, Effect } from "effect";
 import type * as Cause from "effect/Cause";
 
 import type { ExecutionEngine, ExecutionResult } from "@executor-js/execution";
 
+import { captureCauseEffect } from "../observability";
 import { EXECUTION_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";
 
 // The engine's completed-result payload (`ExecuteResult` in codemode-core),
@@ -166,10 +166,12 @@ export const makeExecutionLimitGate = (checkBalance: ExecutionBalanceCheck) => {
         // must never block executions. Reported like `trackExecution` so a
         // billing outage still pages; the error outcome is never cached.
         Effect.catch((error: unknown) =>
-          Effect.sync((): GateDecision => {
-            console.warn("[billing] execution balance check failed open:", error);
-            Sentry.captureException(error);
-            return { blocked: false };
+          Effect.gen(function* () {
+            yield* Effect.sync(() => {
+              console.warn("[billing] execution balance check failed open:", error);
+            });
+            yield* captureCauseEffect(error);
+            return { blocked: false } as const satisfies GateDecision;
           }),
         ),
       );

apps/cloud/src/engine/execution-rate-limit.ts

@@ -14,12 +14,12 @@
 // ---------------------------------------------------------------------------
 
 import { DurableObject, env } from "cloudflare:workers";
-import * as Sentry from "@sentry/cloudflare";
 import { Data, Effect } from "effect";
 import type * as Cause from "effect/Cause";
 
 import type { ExecutionEngine } from "@executor-js/execution";
 
+import { captureCauseEffect } from "../observability";
 import { withPreExecutionGate, type GateDecision } from "./execution-gate";
 import { RATE_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages";
 
@@ -149,10 +149,12 @@ export const makeExecutionRateLimiter = (
         // FAIL OPEN: the backstop must never block executions because its
         // counter is unreachable or slow.
         Effect.catch((error: unknown) =>
-          Effect.sync((): GateDecision => {
-            console.warn("[rate-limit] execution rate limit check failed open:", error);
-            Sentry.captureException(error);
-            return { blocked: false };
+          Effect.gen(function* () {
+            yield* Effect.sync(() => {
+              console.warn("[rate-limit] execution rate limit check failed open:", error);
+            });
+            yield* captureCauseEffect(error);
+            return { blocked: false } as const satisfies GateDecision;
           }),
         ),
       );

apps/cloud/src/env-augment.d.ts

@@ -11,6 +11,8 @@ declare global {
       AXIOM_TRACES_URL?: string;
       AXIOM_TRACES_SAMPLE_RATIO?: string;
       SENTRY_DSN?: string;
+      SENTRY_OTEL_LOG_PAYLOAD?: string;
+      SENTRY_OTEL_VERIFY?: string;
       VITE_PUBLIC_SENTRY_DSN?: string;
       VITE_PUBLIC_POSTHOG_KEY?: string;
       VITE_PUBLIC_POSTHOG_HOST?: string;

apps/cloud/src/extensions/billing/route.ts

@@ -5,7 +5,11 @@ import { autumnHandler } from "autumn-js/backend";
 
 import { WorkOSClient } from "../../auth/workos";
 import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../../auth/organization";
-import { HttpResponseError, isServerError, toErrorServerResponse } from "../../api/error-response";
+import {
+  HttpResponseError,
+  isServerError,
+  toErrorServerResponseEffect,
+} from "../../api/error-response";
 
 type BillingSession = {
   readonly userId: string;
@@ -111,7 +115,7 @@ const handler = Effect.gen(function* () {
     if (isServerError(err)) {
       console.error("[autumn] request failed:", Cause.pretty(err));
     }
-    return Effect.succeed(toErrorServerResponse(err));
+    return toErrorServerResponseEffect(err);
   }),
 );
 

apps/cloud/src/extensions/billing/service.ts

@@ -3,10 +3,11 @@
 // ---------------------------------------------------------------------------
 
 import { env } from "cloudflare:workers";
-import * as Sentry from "@sentry/cloudflare";
 import { Autumn } from "autumn-js";
 import { Context, Data, Effect, Layer } from "effect";
 
+import { captureCauseEffect } from "../../observability";
+
 // ---------------------------------------------------------------------------
 // Errors
 // ---------------------------------------------------------------------------
@@ -72,12 +73,12 @@ const make = Effect.sync(() => {
       ).pipe(
         Effect.catchTag("AutumnError", (error) =>
           Effect.gen(function* () {
-            // Silent billing data loss is worth paging on — autumn.trackExecution
+            // Silent billing data loss is worth paging on: autumn.trackExecution
             // is fire-and-forget so the caller doesn't handle it themselves.
             yield* Effect.sync(() => {
               console.error("[billing] track failed:", error);
-              Sentry.captureException(error);
             });
+            yield* captureCauseEffect(error);
             yield* Effect.annotateCurrentSpan({ "autumn.track.failed": true });
           }),
         ),

apps/cloud/src/mcp/session-durable-object.ts

@@ -68,7 +68,11 @@ import { makeExecutionStack } from "../engine/execution-stack";
 import { CloudMeteredExecutionStackLayer } from "../engine/execution-stack-metered";
 import { AutumnService } from "../extensions/billing/service";
 import { DoTelemetryLive, flushTracerProvider } from "../observability/telemetry";
-import { captureCause as reportCause } from "../observability";
+import {
+  captureCause as reportCause,
+  captureCauseEffect as reportCauseEffect,
+  tagCurrentSentryScopeWithCurrentOtelSpan,
+} from "../observability";
 
 // Re-export the shared types so existing cloud importers
 // (`auth/handlers.ts`, etc.) keep their `../mcp/session-durable-object` path.
@@ -314,6 +318,16 @@ export class McpSessionDOSqlite extends McpAgentSessionDOBase<Env, CloudSessionD
     reportCause(cause);
   }
 
+  protected override captureCauseEffect(
+    cause: Cause.Cause<unknown>,
+  ): Effect.Effect<string | undefined> {
+    return reportCauseEffect(cause);
+  }
+
+  protected override prepareErrorCaptureScope(): Effect.Effect<void> {
+    return Effect.asVoid(tagCurrentSentryScopeWithCurrentOtelSpan);
+  }
+
   // Best-effort export the DO isolate's buffered spans after the RPC settles,
   // so a dying init/handleRequest can ship its own spans (and the exception +
   // stack recorded on them) — not just the worker-side `mcp.do.*` span. Keep it

apps/cloud/src/observability/index.ts

@@ -11,7 +11,9 @@
 // ---------------------------------------------------------------------------
 
 import * as Sentry from "@sentry/cloudflare";
+import type { ErrorEvent, Scope } from "@sentry/cloudflare";
 import { Cause, Effect, Layer } from "effect";
+import type * as Tracer from "effect/Tracer";
 
 import { ErrorCapture } from "@executor-js/api";
 
@@ -21,12 +23,106 @@ import { ErrorCapture } from "@executor-js/api";
 // error. Sentry still receives the full, untruncated cause via
 // `setExtra`; only the dev-console mirror is capped.
 const MAX_CONSOLE_CAUSE_CHARS = 4_000;
+const OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
+const OTEL_SPAN_ID_PATTERN = /^[0-9a-f]{16}$/;
+
+export const OTEL_TRACE_ID_TAG = "otel_trace_id";
+export const OTEL_SPAN_ID_TAG = "otel_span_id";
+export const SENTRY_EVENT_ID_ATTRIBUTE = "sentry.event_id";
+
+export type OtelCorrelationContext = {
+  readonly traceId: string;
+  readonly spanId: string;
+};
 
 const truncate = (s: string): string =>
   s.length <= MAX_CONSOLE_CAUSE_CHARS
     ? s
     : `${s.slice(0, MAX_CONSOLE_CAUSE_CHARS)}\n…[truncated ${s.length - MAX_CONSOLE_CAUSE_CHARS} chars]`;
 
+const validOtelContext = (context: OtelCorrelationContext): boolean =>
+  OTEL_TRACE_ID_PATTERN.test(context.traceId) && OTEL_SPAN_ID_PATTERN.test(context.spanId);
+
+export const otelCorrelationContextFromEffectSpan = (
+  span: Tracer.Span,
+): OtelCorrelationContext | null => {
+  const context = { traceId: span.traceId, spanId: span.spanId };
+  return validOtelContext(context) ? context : null;
+};
+
+export const otelCorrelationContextFromOpenTelemetrySpan = (span: {
+  readonly spanContext: () => { readonly traceId: string; readonly spanId: string };
+}): OtelCorrelationContext | null => {
+  const { traceId, spanId } = span.spanContext();
+  const context = { traceId, spanId };
+  return validOtelContext(context) ? context : null;
+};
+
+export const addOtelCorrelationTags = <T extends { readonly tags?: Record<string, unknown> }>(
+  event: T,
+  context: OtelCorrelationContext | null,
+): T => {
+  if (!context) return event;
+  return {
+    ...event,
+    tags: {
+      ...event.tags,
+      [OTEL_TRACE_ID_TAG]: context.traceId,
+      [OTEL_SPAN_ID_TAG]: context.spanId,
+    },
+  };
+};
+
+export const tagSentryScopeWithOtelContext = (
+  scope: Scope,
+  context: OtelCorrelationContext | null,
+): void => {
+  if (!context) return;
+  scope.setTag(OTEL_TRACE_ID_TAG, context.traceId);
+  scope.setTag(OTEL_SPAN_ID_TAG, context.spanId);
+};
+
+export const tagCurrentSentryScopeWithOtelContext = (
+  context: OtelCorrelationContext | null,
+): void => {
+  tagSentryScopeWithOtelContext(Sentry.getCurrentScope(), context);
+};
+
+const currentOtelContext = Effect.map(
+  Effect.currentSpan,
+  otelCorrelationContextFromEffectSpan,
+).pipe(Effect.orElseSucceed(() => null));
+
+export const tagCurrentSentryScopeWithCurrentOtelSpan: Effect.Effect<OtelCorrelationContext | null> =
+  Effect.map(currentOtelContext, (context) => {
+    tagCurrentSentryScopeWithOtelContext(context);
+    return context;
+  });
+
+export const beforeSendWithOtelCorrelation = (
+  event: ErrorEvent,
+  options?: { readonly logPayload?: boolean },
+): ErrorEvent => {
+  if (options?.logPayload) {
+    console.info(
+      JSON.stringify({
+        event: "sentry_before_send_otel_correlation",
+        sentry_event_id: event.event_id ?? "",
+        otel_trace_id: String(event.tags?.[OTEL_TRACE_ID_TAG] ?? ""),
+        otel_span_id: String(event.tags?.[OTEL_SPAN_ID_TAG] ?? ""),
+      }),
+    );
+  }
+  return event;
+};
+
+export const addCurrentOtelCorrelationTags = <
+  T extends { readonly tags?: Record<string, unknown> },
+>(
+  event: T,
+): Effect.Effect<T> =>
+  Effect.map(currentOtelContext, (context) => addOtelCorrelationTags(event, context));
+
 // Sentry's `captureException` can't serialize Effect's `CauseImpl` (it logs
 // `'CauseImpl' captured as exception with keys: reasons, ~effect/Cause` and
 // drops the real failure). `Cause.squash` isn't enough on its own: when an
@@ -48,21 +144,36 @@ export const sentryPayloadForCause = (
   return { primary: input, pretty: null };
 };
 
-export const captureCause = (input: unknown): string | undefined => {
+export const captureCause = (
+  input: unknown,
+  context: OtelCorrelationContext | null = null,
+): string | undefined => {
   const { primary, pretty } = sentryPayloadForCause(input);
+  tagCurrentSentryScopeWithOtelContext(context);
   return Sentry.captureException(primary, (scope) => {
+    tagSentryScopeWithOtelContext(scope, context);
     if (pretty !== null) scope.setExtra("cause", pretty);
     return scope;
   });
 };
 
+export const captureCauseEffect = (input: unknown): Effect.Effect<string | undefined> =>
+  Effect.gen(function* () {
+    const context = yield* tagCurrentSentryScopeWithCurrentOtelSpan;
+    const eventId = yield* Effect.sync(() => captureCause(input, context));
+    if (eventId && context) {
+      yield* Effect.annotateCurrentSpan(SENTRY_EVENT_ID_ATTRIBUTE, eventId);
+    }
+    return eventId;
+  });
+
 export const ErrorCaptureLive: Layer.Layer<ErrorCapture> = Layer.succeed(
   ErrorCapture,
   ErrorCapture.of({
     captureException: (cause) =>
-      Effect.sync(() => {
+      Effect.gen(function* () {
         console.error("[api] unhandled cause:", truncate(Cause.pretty(cause)));
-        return captureCause(cause) ?? "";
+        return (yield* captureCauseEffect(cause)) ?? "";
       }),
   }),
 );