Skip to content

Commit 59c3e24

Browse files
RhysSullivanRhysSullivan
authored
E2e distributed tracing: suite-owned motel, browser OTLP, end-to-end trace join (#979)
* Install @executor-js/motel for local trace inspection Our fork of kitlangton/motel (UsefulSoftwareCo/motel) with the query- worker serialization fix: trace/span detail endpoints 500ed upstream ('Expected JSON value' — shared span objects tripping Schema.Unknown's isJson guard). Run it with `bun run motel` in e2e/, then E2E_MOTEL=1 sends browser + server spans there during runs. * Run pages link every API call to its distributed trace The browser surface harvests the W3C traceparent the web app already sends on each API request into the run's traces.json (id + offset + url). The runs viewer renders them as a Distributed traces table — one row per call, linking into the local motel's per-trace waterfall — so 'why was that page slow in this recording' goes from the recording to the exact click→server→DB trace in two clicks. Spans exist in motel when the run exported there (E2E_MOTEL=1); ids harvest regardless. * E2e distributed tracing: suite-owned motel, browser OTLP, trace join Every e2e run captures distributed traces. A suite-owned motel (local OTLP store) boots alongside the WorkOS/Autumn emulators — own port, runs-local store wiped per boot, torn down with the suite — and the cloud boot recipe gains an extraEnv seam so the globalsetup points the app's exporter at it (the same endpoint-agnostic layer that ships prod traces to Axiom). The web client exports OTLP spans when the build names an endpoint (VITE_PUBLIC_OTLP_TRACES_URL, proxied same-origin to motel in dev), and the cloud worker's http.server span adopts the caller's W3C traceparent — one trace id from the click in the browser through the worker to the database. Run pages list every harvested trace id with links into motel's waterfall. Also switches the worker's span processor Simple → Batch: per-span synchronous export fetches under the dev worker added enough request latency to flip the approval pause/resume race in e2e; batching removes the class, and the existing ctx.waitUntil(forceFlush) still drains per request in prod. * CLI/MCP calls join the run's distributed-trace ledger The web app's HttpClient sends a traceparent on its own, but mcporter's plain fetch does not — so the terminal half of a session was invisible in traces.json. The MCP surface now wraps the global fetch: every POST to the target's MCP endpoint gets a minted traceparent (the worker and DO already join whatever arrives) and lands in the ledger with the JSON-RPC method or tool name as its label, duration, status, and source: terminal. traces.json becomes a shared append-merge ledger (src/trace-harvest.ts) so the browser surface's harvested entries and the MCP surface's minted ones interleave by wall clock, and the trace rail tags each row with its window (⌨ terminal / ⊕ browser).
1 parent fe5181b commit 59c3e24

14 files changed

Lines changed: 825 additions & 48 deletions

File tree

‎apps/cloud/src/observability/telemetry.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import { trace } from "@opentelemetry/api";
3939
// which workerd does support.
4040
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http/build/esm/platform/browser/index.js";
4141
import { resourceFromAttributes } from "@opentelemetry/resources";
42-
import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base";
42+
import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
4343
import { WebTracerProvider } from "@opentelemetry/sdk-trace-web";
4444
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
4545
import { env } from "cloudflare:workers";
@@ -61,14 +61,21 @@ const ensureGlobalTracerProvider = (): boolean => {
6161
[ATTR_SERVICE_VERSION]: SERVICE_VERSION,
6262
}),
6363
spanProcessors: [
64-
new SimpleSpanProcessor(
64+
// Batch, not Simple: SimpleSpanProcessor issues one synchronous fetch
65+
// per span END — a busy MCP request emits dozens of spans, and the
66+
// serialized export fetches add seconds of latency inside the request
67+
// (enough to flip pause/resume timing in e2e). Batch buffers and
68+
// ships on a timer; `ctx.waitUntil(flushTracerProvider())` at the end
69+
// of each request still drains the buffer before the isolate exits.
70+
new BatchSpanProcessor(
6571
new OTLPTraceExporter({
6672
url: env.AXIOM_TRACES_URL ?? "https://api.axiom.co/v1/traces",
6773
headers: {
6874
Authorization: `Bearer ${env.AXIOM_TOKEN}`,
6975
"X-Axiom-Dataset": env.AXIOM_DATASET ?? "executor-cloud",
7076
},
7177
}),
78+
{ scheduledDelayMillis: 1_000, maxExportBatchSize: 512 },
7279
),
7380
],
7481
});

‎apps/cloud/src/server.ts‎

Lines changed: 45 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { SpanStatusCode, trace } from "@opentelemetry/api";
1+
import { SpanKind, SpanStatusCode, context, trace } from "@opentelemetry/api";
22
import {
33
ATTR_HTTP_REQUEST_METHOD,
44
ATTR_HTTP_RESPONSE_STATUS_CODE,
@@ -72,32 +72,52 @@ const cloudflareHandler: ExportedHandler<Env> = {
7272
return fetchHandler(request, env, ctx);
7373
}
7474
const url = new URL(request.url);
75-
return tracer.startActiveSpan(`http.server ${request.method}`, async (span) => {
76-
span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method);
77-
span.setAttribute(ATTR_URL_FULL, request.url);
78-
span.setAttribute(ATTR_URL_PATH, url.pathname);
79-
span.setAttribute(ATTR_URL_SCHEME, url.protocol.replace(/:$/, ""));
80-
// Adapter boundary: Cloudflare's fetch handler is a Promise-based
81-
// callback and the OTel span lifecycle needs to observe both the
82-
// resolved response and any thrown error before `span.end()`. Sentry's
83-
// outer wrapper still captures the exception; we only mark span status.
84-
// oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary
85-
try {
86-
const response = await fetchHandler(request, env, ctx);
87-
span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, response.status);
88-
if (response.status >= 500) {
75+
// Join the caller's W3C trace when the request carries one — the web UI
76+
// sends traceparent on every API fetch, so the browser's spans and this
77+
// request share one trace id end to end. Same parsing the DO path does
78+
// in session-durable-object.ts.
79+
const traceparentMatch = /^[0-9a-f]{2}-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/.exec(
80+
request.headers.get("traceparent") ?? "",
81+
);
82+
const parentContext = traceparentMatch
83+
? trace.setSpanContext(context.active(), {
84+
traceId: traceparentMatch[1]!,
85+
spanId: traceparentMatch[2]!,
86+
traceFlags: parseInt(traceparentMatch[3]!, 16),
87+
isRemote: true,
88+
})
89+
: context.active();
90+
return tracer.startActiveSpan(
91+
`http.server ${request.method}`,
92+
{ kind: SpanKind.SERVER },
93+
parentContext,
94+
async (span) => {
95+
span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method);
96+
span.setAttribute(ATTR_URL_FULL, request.url);
97+
span.setAttribute(ATTR_URL_PATH, url.pathname);
98+
span.setAttribute(ATTR_URL_SCHEME, url.protocol.replace(/:$/, ""));
99+
// Adapter boundary: Cloudflare's fetch handler is a Promise-based
100+
// callback and the OTel span lifecycle needs to observe both the
101+
// resolved response and any thrown error before `span.end()`. Sentry's
102+
// outer wrapper still captures the exception; we only mark span status.
103+
// oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary
104+
try {
105+
const response = await fetchHandler(request, env, ctx);
106+
span.setAttribute(ATTR_HTTP_RESPONSE_STATUS_CODE, response.status);
107+
if (response.status >= 500) {
108+
span.setStatus({ code: SpanStatusCode.ERROR });
109+
}
110+
return response;
111+
} catch (err) {
89112
span.setStatus({ code: SpanStatusCode.ERROR });
113+
// oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary; preserve original error to Cloudflare runtime
114+
throw err;
115+
} finally {
116+
span.end();
117+
ctx.waitUntil(flushTracerProvider());
90118
}
91-
return response;
92-
} catch (err) {
93-
span.setStatus({ code: SpanStatusCode.ERROR });
94-
// oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary; preserve original error to Cloudflare runtime
95-
throw err;
96-
} finally {
97-
span.end();
98-
ctx.waitUntil(flushTracerProvider());
99-
}
100-
});
119+
},
120+
);
101121
},
102122
};
103123

‎apps/cloud/vite.config.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,15 @@ export default defineConfig(({ mode }) => {
6363
.filter(([key]) => key.startsWith("VITE_PUBLIC_"))
6464
.map(([key, value]) => [`import.meta.env.${key}`, JSON.stringify(value)]),
6565
),
66+
// Browser OTLP spans (VITE_PUBLIC_OTLP_TRACES_URL=/v1/traces, set by the
67+
// e2e global setup) go same-origin and proxy to the local motel server —
68+
// motel serves no CORS headers, so a direct cross-origin post would die
69+
// in preflight. Dev-only; unrouted when nothing listens.
70+
server: {
71+
proxy: {
72+
"/v1/traces": process.env.MOTEL_URL ?? "http://127.0.0.1:27686",
73+
},
74+
},
6675
resolve: { tsconfigPaths: true },
6776
plugins: [
6877
devCrashGuard(),

0 commit comments

Comments
 (0)