E2e distributed tracing: suite-owned motel, browser OTLP, trace join

Every e2e run captures distributed traces. A suite-owned motel (local
OTLP store) boots alongside the WorkOS/Autumn emulators — own port,
runs-local store wiped per boot, torn down with the suite — and the
cloud boot recipe gains an extraEnv seam so the globalsetup points the
app's exporter at it (the same endpoint-agnostic layer that ships prod
traces to Axiom).

The web client exports OTLP spans when the build names an endpoint
(VITE_PUBLIC_OTLP_TRACES_URL, proxied same-origin to motel in dev), and
the cloud worker's http.server span adopts the caller's W3C traceparent
— one trace id from the click in the browser through the worker to the
database. Run pages list every harvested trace id with links into
motel's waterfall.

Also switches the worker's span processor Simple → Batch: per-span
synchronous export fetches under the dev worker added enough request
latency to flip the approval pause/resume race in e2e; batching removes
the class, and the existing ctx.waitUntil(forceFlush) still drains per
request in prod.

Author: RhysSullivan

apps/cloud/src/observability/telemetry.ts

@@ -39,7 +39,7 @@ import { trace } from "@opentelemetry/api";
 // which workerd does support.
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http/build/esm/platform/browser/index.js";
 import { resourceFromAttributes } from "@opentelemetry/resources";
-import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
 import { WebTracerProvider } from "@opentelemetry/sdk-trace-web";
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
 import { env } from "cloudflare:workers";
@@ -61,14 +61,21 @@ const ensureGlobalTracerProvider = (): boolean => {
       [ATTR_SERVICE_VERSION]: SERVICE_VERSION,
     }),
     spanProcessors: [
-      new SimpleSpanProcessor(
+      // Batch, not Simple: SimpleSpanProcessor issues one synchronous fetch
+      // per span END — a busy MCP request emits dozens of spans, and the
+      // serialized export fetches add seconds of latency inside the request
+      // (enough to flip pause/resume timing in e2e). Batch buffers and
+      // ships on a timer; `ctx.waitUntil(flushTracerProvider())` at the end
+      // of each request still drains the buffer before the isolate exits.
+      new BatchSpanProcessor(
         new OTLPTraceExporter({
           url: env.AXIOM_TRACES_URL ?? "https://api.axiom.co/v1/traces",
           headers: {
             Authorization: `Bearer ${env.AXIOM_TOKEN}`,
             "X-Axiom-Dataset": env.AXIOM_DATASET ?? "executor-cloud",
           },
         }),
+        { scheduledDelayMillis: 1_000, maxExportBatchSize: 512 },
       ),
     ],
   });

apps/cloud/src/server.ts

@@ -1,4 +1,4 @@
-import { SpanStatusCode, trace } from "@opentelemetry/api";
+import { SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";
 import {
   ATTR_HTTP_REQUEST_METHOD,
   ATTR_HTTP_RESPONSE_STATUS_CODE,
@@ -72,32 +72,52 @@ const cloudflareHandler: ExportedHandler<Env> = {
       return fetchHandler(request, env, ctx);
     }
     const url = new URL(request.url);
-    return tracer.startActiveSpan(`http.server ${request.method}`, async (span) => {
-      span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method);
-      span.setAttribute(ATTR_URL_FULL, request.url);
-      span.setAttribute(ATTR_URL_PATH, url.pathname);
-      span.setAttribute(ATTR_URL_SCHEME, url.protocol.replace(/:$/, ""));
-      // Adapter boundary: Cloudflare's fetch handler is a Promise-based
-      // callback and the OTel span lifecycle needs to observe both the
-      // resolved response and any thrown error before `span.end()`. Sentry's
-      // outer wrapper still captures the exception; we only mark span status.
-      // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary
-      try {
-        const response = await fetchHandler(request, env, ctx);
-        span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, response.status);
-        if (response.status >= 500) {
+    // Join the caller's W3C trace when the request carries one — the web UI
+    // sends traceparent on every API fetch, so the browser's spans and this
+    // request share one trace id end to end. Same parsing the DO path does
+    // in session-durable-object.ts.
+    const traceparentMatch = /^[0-9a-f]{2}-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/.exec(
+      request.headers.get("traceparent") ?? "",
+    );
+    const parentContext = traceparentMatch
+      ? trace.setSpanContext(context.active(), {
+          traceId: traceparentMatch[1]!,
+          spanId: traceparentMatch[2]!,
+          traceFlags: parseInt(traceparentMatch[3]!, 16),
+          isRemote: true,
+        })
+      : context.active();
+    return tracer.startActiveSpan(
+      `http.server ${request.method}`,
+      { kind: SpanKind.SERVER },
+      parentContext,
+      async (span) => {
+        span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method);
+        span.setAttribute(ATTR_URL_FULL, request.url);
+        span.setAttribute(ATTR_URL_PATH, url.pathname);
+        span.setAttribute(ATTR_URL_SCHEME, url.protocol.replace(/:$/, ""));
+        // Adapter boundary: Cloudflare's fetch handler is a Promise-based
+        // callback and the OTel span lifecycle needs to observe both the
+        // resolved response and any thrown error before `span.end()`. Sentry's
+        // outer wrapper still captures the exception; we only mark span status.
+        // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary
+        try {
+          const response = await fetchHandler(request, env, ctx);
+          span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, response.status);
+          if (response.status >= 500) {
+            span.setStatus({ code: SpanStatusCode.ERROR });
+          }
+          return response;
+        } catch (err) {
           span.setStatus({ code: SpanStatusCode.ERROR });
+          // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary; preserve original error to Cloudflare runtime
+          throw err;
+        } finally {
+          span.end();
+          ctx.waitUntil(flushTracerProvider());
         }
-        return response;
-      } catch (err) {
-        span.setStatus({ code: SpanStatusCode.ERROR });
-        // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary; preserve original error to Cloudflare runtime
-        throw err;
-      } finally {
-        span.end();
-        ctx.waitUntil(flushTracerProvider());
-      }
-    });
+      },
+    );
   },
 };
 

apps/cloud/vite.config.ts

@@ -20,7 +20,10 @@ const devCrashGuard = (): Plugin => {
     if (installed) return;
     installed = true;
     process.on("uncaughtException", (err, origin) => {
-      console.error(`[dev-crash-guard] uncaughtException (origin=${origin}):`, err);
+      console.error(
+        `[dev-crash-guard] uncaughtException (origin=${origin}):`,
+        err,
+      );
     });
     process.on("unhandledRejection", (reason, promise) => {
       console.error("[dev-crash-guard] unhandledRejection:", reason, promise);
@@ -39,7 +42,9 @@ const loadWranglerPublicVars = () => {
     { hideWarnings: true },
   );
   return Object.fromEntries(
-    Object.entries(wranglerConfig.vars ?? {}).filter(([key]) => key.startsWith("VITE_PUBLIC_")),
+    Object.entries(wranglerConfig.vars ?? {}).filter(([key]) =>
+      key.startsWith("VITE_PUBLIC_"),
+    ),
   );
 };
 
@@ -61,8 +66,20 @@ export default defineConfig(({ mode }) => {
     define: Object.fromEntries(
       Object.entries(publicEnv)
         .filter(([key]) => key.startsWith("VITE_PUBLIC_"))
-        .map(([key, value]) => [`import.meta.env.${key}`, JSON.stringify(value)]),
+        .map(([key, value]) => [
+          `import.meta.env.${key}`,
+          JSON.stringify(value),
+        ]),
     ),
+    // Browser OTLP spans (VITE_PUBLIC_OTLP_TRACES_URL=/v1/traces, set by the
+    // e2e global setup) go same-origin and proxy to the local motel server —
+    // motel serves no CORS headers, so a direct cross-origin post would die
+    // in preflight. Dev-only; unrouted when nothing listens.
+    server: {
+      proxy: {
+        "/v1/traces": process.env.MOTEL_URL ?? "http://127.0.0.1:27686",
+      },
+    },
     resolve: { tsconfigPaths: true },
     plugins: [
       devCrashGuard(),
@@ -81,7 +98,11 @@ export default defineConfig(({ mode }) => {
           virtualRouteConfig: rootRoute("__root.tsx", [
             ...consoleRoutes({
               dir: "../../../../packages/react/src/routes",
-              exclude: ["/secrets", "/resume/$executionId", "/plugins/$pluginId/$"],
+              exclude: [
+                "/secrets",
+                "/resume/$executionId",
+                "/plugins/$pluginId/$",
+              ],
             }),
             physical("", "app"),
           ]),

e2e/package.json

@@ -15,7 +15,7 @@
     "serve": "bun scripts/rebuild-viewer.ts && bun scripts/serve.ts",
     "typecheck": "tsc --noEmit",
 "test:desktop": "vitest run --project desktop",
-    "motel": "motel"
+    "motel": "MOTEL_OTEL_BASE_URL=http://127.0.0.1:4796 MOTEL_OTEL_DB_PATH=runs/.motel/telemetry.sqlite motel server"
   },
   "dependencies": {
     "@executor-js/api": "workspace:*",

e2e/setup/cloud.boot.ts

@@ -12,7 +12,9 @@ import { createEmulator } from "@executor-js/emulate";
 
 import { bootProcesses, waitForHttp } from "./boot";
 
-export const cloudDir = fileURLToPath(new URL("../../apps/cloud/", import.meta.url));
+export const cloudDir = fileURLToPath(
+  new URL("../../apps/cloud/", import.meta.url),
+);
 
 export interface CloudBootOptions {
   readonly cloudPort: number;
@@ -35,6 +37,8 @@ export interface CloudBootOptions {
   /** Wipe the dev DB before boot (hermetic). Default true. */
   readonly fresh?: boolean;
   readonly logFile?: string;
+  /** Extra env for the app's dev stack (e.g. the suite's OTLP exporter). */
+  readonly extraEnv?: Record<string, string>;
 }
 
 export interface CloudBooted {
@@ -44,7 +48,9 @@ export interface CloudBooted {
   readonly autumnUrl: string;
 }
 
-export const bootCloud = async (options: CloudBootOptions): Promise<CloudBooted> => {
+export const bootCloud = async (
+  options: CloudBootOptions,
+): Promise<CloudBooted> => {
   // Fresh dev DB per boot — the WorkOS emulator mints org ids from a
   // per-process counter, so a persisted DB from a previous invocation
   // collides with the new boot's ids (identities land in polluted orgs /
@@ -60,7 +66,10 @@ export const bootCloud = async (options: CloudBootOptions): Promise<CloudBooted>
     port: options.workosPort,
     ...(options.workosPublicUrl ? { baseUrl: options.workosPublicUrl } : {}),
   });
-  const autumn = await createEmulator({ service: "autumn", port: options.autumnPort });
+  const autumn = await createEmulator({
+    service: "autumn",
+    port: options.autumnPort,
+  });
 
   const workosUrl = options.workosPublicUrl ?? workos.url;
   const env = {
@@ -72,7 +81,8 @@ export const bootCloud = async (options: CloudBootOptions): Promise<CloudBooted>
     WORKOS_CLIENT_ID: options.workosClientId,
     WORKOS_COOKIE_PASSWORD: options.cookiePassword,
     AUTUMN_SECRET_KEY: "am_test_emulate",
-    ENCRYPTION_KEY: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+    ENCRYPTION_KEY:
+      "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
     DATABASE_URL: `postgresql://postgres:postgres@127.0.0.1:${options.dbPort}/postgres`,
     EXECUTOR_DIRECT_DATABASE_URL: "true",
     CLOUDFLARE_INCLUDE_PROCESS_ENV: "true",
@@ -87,6 +97,7 @@ export const bootCloud = async (options: CloudBootOptions): Promise<CloudBooted>
     // Vite rejects unknown Host headers; allow the public hostname when a
     // proxy fronts the app.
     __VITE_ADDITIONAL_SERVER_ALLOWED_HOSTS: new URL(options.publicUrl).hostname,
+    ...options.extraEnv,
   };
 
   const procs = bootProcesses(

e2e/setup/cloud.globalsetup.ts

@@ -7,6 +7,7 @@ import { claimPorts } from "../src/ports";
 import { E2E_COOKIE_PASSWORD, E2E_WORKOS_CLIENT_ID } from "../targets/cloud";
 import { waitForHttp } from "./boot";
 import { bootCloud } from "./cloud.boot";
+import { bootMotel, motelExporterEnv } from "./motel";
 
 export default async function setup(): Promise<(() => Promise<void>) | void> {
   if (process.env.E2E_CLOUD_URL) {
@@ -25,6 +26,10 @@ export default async function setup(): Promise<(() => Promise<void>) | void> {
     { envVar: "E2E_AUTUMN_EMULATOR_PORT", offset: 3, label: "Autumn emulator" },
   ]);
 
+  // Suite-owned trace store — every run captures distributed traces.
+  const motel = await bootMotel();
+
+  const publicUrl = `http://127.0.0.1:${ports.E2E_CLOUD_PORT!}`;
   let booted;
   try {
     booted = await bootCloud({
@@ -34,14 +39,21 @@ export default async function setup(): Promise<(() => Promise<void>) | void> {
       autumnPort: ports.E2E_AUTUMN_EMULATOR_PORT!,
       workosClientId: E2E_WORKOS_CLIENT_ID,
       cookiePassword: E2E_COOKIE_PASSWORD,
-      publicUrl: `http://127.0.0.1:${ports.E2E_CLOUD_PORT!}`,
+      publicUrl,
+      // Server + browser spans → the suite's motel. The app's exporter is
+      // endpoint-agnostic, so the same layer that ships prod traces to
+      // Axiom ships e2e traces to the suite store — "why was that page
+      // slow" gets a span waterfall, not a guess.
+      extraEnv: motelExporterEnv(motel, publicUrl),
     });
   } catch (error) {
+    await motel?.teardown();
     await release();
     throw error;
   }
   return async () => {
     await booted.teardown();
+    await motel?.teardown();
     await release();
   };
 }

e2e/setup/motel.ts

@@ -0,0 +1,74 @@
+// Suite-owned motel: a fresh local OTLP store booted alongside the target's
+// dev stack (same pattern as the WorkOS/Autumn emulators) so EVERY run
+// captures distributed traces — hermetically, in CI too, with no dependence
+// on a machine-global daemon whose health or leftover data could leak into
+// results. DB lives under runs/.motel so the suite's evidence stays with
+// the suite; wiped per boot like the target's dev DB.
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { bootProcesses, waitForHttp, type BootedProcesses } from "./boot";
+
+export const MOTEL_PORT = 4796;
+export const MOTEL_URL = `http://127.0.0.1:${MOTEL_PORT}`;
+
+const e2eDir = fileURLToPath(new URL("..", import.meta.url));
+
+export interface SuiteMotel {
+  readonly url: string;
+  readonly teardown: () => Promise<void>;
+}
+
+/** Boot the suite's motel server. Never fails the suite: if the binary or
+ *  the port is unavailable, tracing is simply off (null) and targets skip
+ *  the exporter env. */
+export const bootMotel = async (): Promise<SuiteMotel | null> => {
+  const dataDir = join(e2eDir, "runs", ".motel");
+  rmSync(dataDir, { recursive: true, force: true });
+  mkdirSync(dataDir, { recursive: true });
+
+  let procs: BootedProcesses | null = null;
+  // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: optional infrastructure; a motel-less host still runs the suite
+  try {
+    procs = bootProcesses(
+      [
+        {
+          cmd: "bunx",
+          args: ["motel", "server"],
+          cwd: e2eDir,
+          env: {
+            MOTEL_OTEL_BASE_URL: MOTEL_URL,
+            MOTEL_OTEL_DB_PATH: join(dataDir, "telemetry.sqlite"),
+          },
+        },
+      ],
+      { label: "motel" },
+    );
+    await waitForHttp(`${MOTEL_URL}/api/health`);
+    console.log(`[e2e] traces → suite motel at ${MOTEL_URL}`);
+    return { url: MOTEL_URL, teardown: procs.teardown };
+  } catch (error) {
+    console.warn(`[e2e] motel unavailable, tracing off: ${String(error)}`);
+    await procs?.teardown();
+    return null;
+  }
+};
+
+/** Exporter env for a target's dev stack: server spans via the app's
+ *  endpoint-agnostic Axiom exporter, browser spans via packages/react's
+ *  OTLP tracer (same-origin /v1/traces, proxied by the dev server — motel
+ *  serves no CORS headers). */
+export const motelExporterEnv = (
+  motel: SuiteMotel | null,
+  appBaseUrl: string,
+): Record<string, string> =>
+  motel
+    ? {
+        AXIOM_TRACES_URL: `${motel.url}/v1/traces`,
+        AXIOM_TOKEN: "motel-local",
+        AXIOM_DATASET: "executor-e2e",
+        VITE_PUBLIC_OTLP_TRACES_URL: `${appBaseUrl}/v1/traces`,
+        MOTEL_URL: motel.url,
+      }
+    : {};