feat(cloudflare): Split alarms into multiple traces and link them

Author: JPeer264

dev-packages/e2e-tests/test-applications/cloudflare-workers/src/index.ts

@@ -19,6 +19,13 @@ class MyDurableObjectBase extends DurableObject<Env> {
     throw new Error('Should be recorded in Sentry.');
   }
 
+  async alarm(): Promise<void> {
+    const action = await this.ctx.storage.get<string>('alarm-action');
+    if (action === 'throw') {
+      throw new Error('Alarm error captured by Sentry');
+    }
+  }
+
   async fetch(request: Request) {
     const url = new URL(request.url);
     switch (url.pathname) {
@@ -32,6 +39,12 @@ class MyDurableObjectBase extends DurableObject<Env> {
         this.ctx.acceptWebSocket(server);
         return new Response(null, { status: 101, webSocket: client });
       }
+      case '/setAlarm': {
+        const action = url.searchParams.get('action') || 'succeed';
+        await this.ctx.storage.put('alarm-action', action);
+        await this.ctx.storage.setAlarm(Date.now() + 500);
+        return new Response('Alarm set');
+      }
       case '/storage/put': {
         await this.ctx.storage.put('test-key', 'test-value');
         return new Response('Stored');

dev-packages/e2e-tests/test-applications/cloudflare-workers/tests/index.test.ts

@@ -99,3 +99,47 @@ test('Storage operations create spans in Durable Object transactions', async ({
   expect(putSpan?.data?.['db.system.name']).toBe('cloudflare.durable_object.storage');
   expect(putSpan?.data?.['db.operation.name']).toBe('put');
 });
+
+test.describe('Alarm instrumentation', () => {
+  test.describe.configure({ mode: 'serial' });
+
+  test('captures error from alarm handler', async ({ baseURL }) => {
+    const errorWaiter = waitForError('cloudflare-workers', event => {
+      return event.exception?.values?.[0]?.value === 'Alarm error captured by Sentry';
+    });
+
+    const response = await fetch(`${baseURL}/pass-to-object/setAlarm?action=throw`);
+    expect(response.status).toBe(200);
+
+    const event = await errorWaiter;
+    expect(event.exception?.values?.[0]?.mechanism?.type).toBe('auto.faas.cloudflare.durable_object');
+  });
+
+  test('creates a transaction for alarm with new trace linked to setAlarm', async ({ baseURL }) => {
+    const setAlarmTransactionWaiter = waitForTransaction('cloudflare-workers', event => {
+      return event.spans?.some(span => span.description?.includes('storage_setAlarm')) ?? false;
+    });
+
+    const alarmTransactionWaiter = waitForTransaction('cloudflare-workers', event => {
+      return event.transaction === 'alarm' && event.contexts?.trace?.op === 'function';
+    });
+
+    const response = await fetch(`${baseURL}/pass-to-object/setAlarm`);
+    expect(response.status).toBe(200);
+
+    const setAlarmTransaction = await setAlarmTransactionWaiter;
+    const alarmTransaction = await alarmTransactionWaiter;
+
+    // Alarm creates a transaction with correct attributes
+    expect(alarmTransaction.contexts?.trace?.op).toBe('function');
+    expect(alarmTransaction.contexts?.trace?.origin).toBe('auto.faas.cloudflare.durable_object');
+
+    // Alarm starts a new trace (different trace ID from the request that called setAlarm)
+    expect(alarmTransaction.contexts?.trace?.trace_id).not.toBe(setAlarmTransaction.contexts?.trace?.trace_id);
+
+    // Alarm links to the trace that called setAlarm via sentry.previous_trace attribute
+    const previousTrace = alarmTransaction.contexts?.trace?.data?.['sentry.previous_trace'];
+    expect(previousTrace).toBeDefined();
+    expect(previousTrace).toContain(setAlarmTransaction.contexts?.trace?.trace_id);
+  });
+});

packages/cloudflare/src/durableobject.ts

@@ -82,7 +82,11 @@ export function instrumentDurableObjectWithSentry<
       }
 
       if (obj.alarm && typeof obj.alarm === 'function') {
-        obj.alarm = wrapMethodWithSentry({ options, context, spanName: 'alarm' }, obj.alarm);
+        // Alarms are independent invocations, so we start a new trace and link to the previous alarm
+        obj.alarm = wrapMethodWithSentry(
+          { options, context, spanName: 'alarm', spanOp: 'function', startNewTrace: true, linkPreviousTrace: true },
+          obj.alarm,
+        );
       }
 
       if (obj.webSocketMessage && typeof obj.webSocketMessage === 'function') {

packages/cloudflare/src/instrumentations/instrumentDurableObjectStorage.ts

@@ -1,20 +1,31 @@
 import type { DurableObjectStorage } from '@cloudflare/workers-types';
-import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, startSpan } from '@sentry/core';
+import { isThenable, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, startSpan } from '@sentry/core';
+import { storeSpanContext } from '../utils/traceLinks';
 
-const STORAGE_METHODS_TO_INSTRUMENT = ['get', 'put', 'delete', 'list'] as const;
+const STORAGE_METHODS_TO_INSTRUMENT = ['get', 'put', 'delete', 'list', 'setAlarm', 'getAlarm', 'deleteAlarm'] as const;
 
 type StorageMethod = (typeof STORAGE_METHODS_TO_INSTRUMENT)[number];
 
+type WaitUntil = (promise: Promise<unknown>) => void;
+
 /**
  * Instruments DurableObjectStorage methods with Sentry spans.
  *
  * Wraps the following async methods:
  * - get, put, delete, list (KV API)
+ * - setAlarm, getAlarm, deleteAlarm (Alarm API)
+ *
+ * When setAlarm is called, it also stores the current span context so that when
+ * the alarm fires later, it can link back to the trace that called setAlarm.
  *
  * @param storage - The DurableObjectStorage instance to instrument
+ * @param waitUntil - Optional waitUntil function to defer span context storage
  * @returns An instrumented DurableObjectStorage instance
  */
-export function instrumentDurableObjectStorage(storage: DurableObjectStorage): DurableObjectStorage {
+export function instrumentDurableObjectStorage(
+  storage: DurableObjectStorage,
+  waitUntil?: WaitUntil,
+): DurableObjectStorage {
   return new Proxy(storage, {
     get(target, prop, _receiver) {
       // Use `target` as the receiver instead of the proxy (`_receiver`).
@@ -46,7 +57,33 @@ export function instrumentDurableObjectStorage(storage: DurableObjectStorage): D
             },
           },
           () => {
-            return (original as (...args: unknown[]) => unknown).apply(target, args);
+            const teardown = async (): Promise<void> => {
+              // When setAlarm is called, store the current span context so that when the alarm
+              // fires later, it can link back to the trace that called setAlarm.
+              // We use the original (uninstrumented) storage (target) to avoid creating a span
+              // for this internal operation. The storage is deferred via waitUntil to not block.
+              if (methodName === 'setAlarm') {
+                await storeSpanContext(target, 'alarm');
+              }
+            };
+
+            const result = (original as (...args: unknown[]) => unknown).apply(target, args);
+
+            if (!isThenable(result)) {
+              waitUntil?.(teardown());
+
+              return result;
+            }
+
+            return result.then(
+              res => {
+                waitUntil?.(teardown());
+                return res;
+              },
+              e => {
+                throw e;
+              },
+            );
           },
         );
       };

packages/cloudflare/src/utils/instrumentContext.ts

@@ -41,13 +41,14 @@ export function instrumentContext<T extends ContextType>(ctx: T): T {
   // If so, wrap the storage with instrumentation
   if ('storage' in ctx && ctx.storage) {
     const originalStorage = ctx.storage;
+    const waitUntil = 'waitUntil' in ctx && typeof ctx.waitUntil === 'function' ? ctx.waitUntil.bind(ctx) : undefined;
     let instrumentedStorage: DurableObjectStorage | undefined;
     descriptors.storage = {
       configurable: true,
       enumerable: true,
       get: () => {
         if (!instrumentedStorage) {
-          instrumentedStorage = instrumentDurableObjectStorage(originalStorage);
+          instrumentedStorage = instrumentDurableObjectStorage(originalStorage, waitUntil);
         }
         return instrumentedStorage;
       },

packages/cloudflare/src/utils/traceLinks.ts

@@ -0,0 +1,84 @@
+import type { DurableObjectStorage } from '@cloudflare/workers-types';
+import { TraceFlags } from '@opentelemetry/api';
+import { getActiveSpan } from '@sentry/core';
+
+/** Storage key prefix for the span context that links consecutive method invocations */
+const SENTRY_TRACE_LINK_KEY_PREFIX = '__SENTRY_TRACE_LINK__';
+
+/** Stored span context for creating span links */
+export interface StoredSpanContext {
+  traceId: string;
+  spanId: string;
+  sampled: boolean;
+}
+
+/** Span link structure for connecting traces */
+export interface SpanLink {
+  context: {
+    traceId: string;
+    spanId: string;
+    traceFlags: number;
+  };
+  attributes?: Record<string, string>;
+}
+
+/**
+ * Gets the storage key for a specific method's trace link.
+ */
+export function getTraceLinkKey(methodName: string): string {
+  return `${SENTRY_TRACE_LINK_KEY_PREFIX}${methodName}`;
+}
+
+/**
+ * Stores the current span context in Durable Object storage for trace linking.
+ * Uses the original uninstrumented storage to avoid creating spans for internal operations.
+ * Errors are silently ignored to prevent internal storage failures from propagating to user code.
+ */
+export async function storeSpanContext(originalStorage: DurableObjectStorage, methodName: string): Promise<void> {
+  try {
+    const activeSpan = getActiveSpan();
+    if (activeSpan) {
+      const spanContext = activeSpan.spanContext();
+      const storedContext: StoredSpanContext = {
+        traceId: spanContext.traceId,
+        spanId: spanContext.spanId,
+        sampled: spanContext.traceFlags === TraceFlags.SAMPLED,
+      };
+      await originalStorage.put(getTraceLinkKey(methodName), storedContext);
+    }
+  } catch {
+    // Silently ignore storage errors to prevent internal failures from affecting user code
+  }
+}
+
+/**
+ * Retrieves a stored span context from Durable Object storage.
+ */
+export async function getStoredSpanContext(
+  originalStorage: DurableObjectStorage,
+  methodName: string,
+): Promise<StoredSpanContext | undefined> {
+  try {
+    return await originalStorage.get<StoredSpanContext>(getTraceLinkKey(methodName));
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Builds span links from a stored span context.
+ */
+export function buildSpanLinks(storedContext: StoredSpanContext): SpanLink[] {
+  return [
+    {
+      context: {
+        traceId: storedContext.traceId,
+        spanId: storedContext.spanId,
+        traceFlags: storedContext.sampled ? TraceFlags.SAMPLED : TraceFlags.NONE,
+      },
+      attributes: {
+        'sentry.link.type': 'previous_trace',
+      },
+    },
+  ];
+}