perf(webapp): shrink run trace loader payload and add trace span cap controls (#3906)

## Summary

The run trace page loader serialized every span's raw OTel events (with
full properties) into the response, even though the tree UI only renders
the derived `timelineEvents` and the span detail panel refetches what it
needs. On event-heavy traces that inflated both the loader payload and
the server-side heap copies built per request. This PR keeps raw span
events server-side and pairs that with a few related trace-view
improvements:

- A new optional `TRACE_VIEW_EMERGENCY_SPAN_CAP` env var (unset by
default) clamps the trace summary and detailed trace summary span limits
on both event store paths, including the public run trace endpoint, so
operators can bound trace query sizes in one place without retuning the
per-store limits.
- The TreeView virtualizer resolved every rendered row with a linear
scan over the whole tree (and `getNodeProps` did the same via
`findIndex`); rows now resolve through memoized id lookup maps, which
matters once traces reach tens of thousands of spans.
- The run stream SSE lookup now applies the same organization membership
scoping as the rest of the run page presenters, for consistency.

Behavior is unchanged by default: the trace tree renders from the same
`timelineEvents` it always has, and the new cap only takes effect when
set.

Author: ericallam

.server-changes/trace-page-payload-diet.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Shrinks the run trace page loader payload by keeping raw span events server-side and makes large trace trees render more efficiently. Also adds an optional `TRACE_VIEW_EMERGENCY_SPAN_CAP` env var that clamps trace summary and detailed summary span limits on both event store paths.

apps/webapp/app/components/primitives/TreeView/TreeView.tsx

@@ -1,6 +1,14 @@
 import { VirtualItem, Virtualizer, useVirtualizer } from "@tanstack/react-virtual";
 import { motion } from "framer-motion";
-import { MutableRefObject, RefObject, useCallback, useEffect, useReducer, useRef } from "react";
+import {
+  MutableRefObject,
+  RefObject,
+  useCallback,
+  useEffect,
+  useMemo,
+  useReducer,
+  useRef,
+} from "react";
 import { cn } from "~/utils/cn";
 import { NodeState, NodesState, reducer } from "./reducer";
 import { concreteStateFromInput, selectedIdFromState } from "./utils";
@@ -47,6 +55,16 @@ export function TreeView<TData>({
 
   const virtualItems = virtualizer.getVirtualItems();
 
+  // id -> node lookup so each virtual row resolves in O(1) instead of
+  // scanning the whole tree per row.
+  const nodesById = useMemo(() => {
+    const map = new Map<string, FlatTreeItem<TData>>();
+    for (const node of tree) {
+      map.set(node.id, node);
+    }
+    return map;
+  }, [tree]);
+
   const scrollCallback = useCallback(
     (event: Event) => {
       if (!onScroll) return;
@@ -99,7 +117,7 @@ export function TreeView<TData>({
           }}
         >
           {virtualItems.map((virtualItem) => {
-            const node = tree.find((node) => node.id === virtualItem.key);
+            const node = nodesById.get(virtualItem.key as string);
             if (!node) return null;
             const state = nodes[node.id];
             if (!state) return null;
@@ -197,6 +215,16 @@ export function useTree<TData, TFilterValue>({
     concreteStateFromInput({ tree, selectedId, collapsedIds, filter })
   );
 
+  // id -> index lookup so getNodeProps resolves in O(1) instead of scanning
+  // the whole tree per rendered row.
+  const treeIndexById = useMemo(() => {
+    const map = new Map<string, number>();
+    tree.forEach((node, index) => {
+      map.set(node.id, index);
+    });
+    return map;
+  }, [tree]);
+
   //sync external selectedId prop into internal state
   useEffect(() => {
     const internalSelectedId = selectedIdFromState(state.nodes);
@@ -497,7 +525,7 @@ export function useTree<TData, TFilterValue>({
     (id: string) => {
       const node = state.nodes[id];
       if (!node) return {};
-      const treeItemIndex = tree.findIndex((node) => node.id === id);
+      const treeItemIndex = treeIndexById.get(id) ?? -1;
       const treeItem = tree[treeItemIndex];
       return {
         "aria-expanded": node.expanded,
@@ -506,7 +534,7 @@ export function useTree<TData, TFilterValue>({
         tabIndex: node.selected ? -1 : undefined,
       };
     },
-    [state]
+    [state, treeIndexById]
   );
 
   return {

apps/webapp/app/env.server.ts

@@ -726,6 +726,9 @@ const EnvironmentSchema = z
     MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
     MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
     MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000),
+    // Emergency circuit breaker: when set, clamps the trace summary and detailed
+    // summary span limits on both event store paths to this value. Unset = disabled.
+    TRACE_VIEW_EMERGENCY_SPAN_CAP: z.coerce.number().int().positive().optional(),
     TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB
     BATCH_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().optional(), // Defaults to TASK_PAYLOAD_OFFLOAD_THRESHOLD if not set
     TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -255,12 +255,16 @@ export class RunPresenter {
             linkedRunIdBySpanId[n.id] = n.runId;
           }
 
+          // Raw span events are only needed server-side (to derive timelineEvents);
+          // keep them out of the serialized loader payload.
+          const { events: spanEvents, ...data } = n.data;
+
           return {
             ...n,
             data: {
-              ...n.data,
+              ...data,
               timelineEvents: createTimelineSpanEventsFromSpanEvents(
-                n.data.events,
+                spanEvents,
                 user?.admin ?? false,
                 treeRootStartTimeMs
               ),

apps/webapp/app/presenters/v3/RunStreamPresenter.server.ts

@@ -1,5 +1,6 @@
 import { type PrismaClient, prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
+import { requireUserId } from "~/services/session.server";
 import { singleton } from "~/utils/singleton";
 import { ABORT_REASON_SEND_ERROR, createSSELoader, SendFunction } from "~/utils/sse";
 import { throttle } from "~/utils/throttle";
@@ -30,9 +31,23 @@ export class RunStreamPresenter {
           throw new Response("Missing runParam", { status: 400 });
         }
 
+        const userId = await requireUserId(context.request);
+
+        // Scope the lookup to organizations the requesting user is a member
+        // of, matching RunPresenter's run lookup. Unauthorized and missing
+        // runs are indistinguishable (both 404).
         const run = await prismaClient.taskRun.findFirst({
           where: {
             friendlyId: runFriendlyId,
+            project: {
+              organization: {
+                members: {
+                  some: {
+                    userId,
+                  },
+                },
+              },
+            },
           },
           select: {
             traceId: true,
@@ -51,7 +66,15 @@ export class RunStreamPresenter {
           if (buffer) {
             try {
               const entry = await buffer.getEntry(runFriendlyId);
-              if (entry) {
+              // Same membership scoping as the PG lookup above — the buffer
+              // entry carries the owning org's id.
+              const isMember = entry
+                ? (await prismaClient.orgMember.findFirst({
+                    where: { organizationId: entry.orgId, userId },
+                    select: { id: true },
+                  })) !== null
+                : false;
+              if (entry && isMember) {
                 // Go through the webapp wrapper so this read-side module
                 // shares a single deserialisation path with readFallback —
                 // see the contract comment in syntheticRedirectInfo.server.ts.

apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts

@@ -2,6 +2,7 @@ import { ClickHouse } from "@internal/clickhouse";
 import { createHash } from "crypto";
 import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server";
 import { env } from "~/env.server";
+import { clampToEmergencySpanCap } from "~/v3/eventRepository/emergencySpanCap.server";
 import { singleton } from "~/utils/singleton";
 import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server";
 import { type IEventRepository } from "~/v3/eventRepository/eventRepository.types";
@@ -533,9 +534,12 @@ function buildEventRepository(store: string, clickhouse: ClickHouse): Clickhouse
         clickhouse,
         batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
         flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
-        maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
-        maximumTraceDetailedSummaryViewCount:
-          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
+        maximumTraceSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT
+        ),
+        maximumTraceDetailedSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT
+        ),
         maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
         insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
         waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",
@@ -557,9 +561,12 @@ function buildEventRepository(store: string, clickhouse: ClickHouse): Clickhouse
         clickhouse: clickhouse,
         batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE,
         flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS,
-        maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT,
-        maximumTraceDetailedSummaryViewCount:
-          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT,
+        maximumTraceSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT
+        ),
+        maximumTraceDetailedSummaryViewCount: clampToEmergencySpanCap(
+          env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT
+        ),
         maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING,
         insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY,
         waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1",

apps/webapp/app/utils/timelineSpanEvents.ts

@@ -117,7 +117,6 @@ export function createTimelineSpanEventsFromSpanEvents(
       offset,
       timestamp,
       duration,
-      properties: spanEvent.properties,
       helpText: getHelpTextForEvent(name),
       markerVariant,
       lineVariant: "light" as const,

apps/webapp/app/v3/eventRepository/emergencySpanCap.server.ts

@@ -0,0 +1,8 @@
+import { env } from "~/env.server";
+
+// Emergency circuit breaker for trace views: when TRACE_VIEW_EMERGENCY_SPAN_CAP
+// is set, clamp a trace summary span limit to it. Unset = no clamping.
+export function clampToEmergencySpanCap(limit: number): number {
+  const cap = env.TRACE_VIEW_EMERGENCY_SPAN_CAP;
+  return cap === undefined ? limit : Math.min(limit, cap);
+}

apps/webapp/app/v3/mollifier/syntheticTrace.server.ts

@@ -44,11 +44,14 @@ export function buildSyntheticTraceForBufferedRun(run: SyntheticRun) {
         const offset = millisecondsToNanoseconds(
           n.data.startTime.getTime() - treeRootStartTimeMs
         );
+        // Mirror RunPresenter: raw span events stay server-side, only
+        // timelineEvents ship to the client.
+        const { events: spanEvents, ...data } = n.data;
         return {
           ...n,
           data: {
-            ...n.data,
-            timelineEvents: createTimelineSpanEventsFromSpanEvents(n.data.events, false, treeRootStartTimeMs),
+            ...data,
+            timelineEvents: createTimelineSpanEventsFromSpanEvents(spanEvents, false, treeRootStartTimeMs),
             duration: n.data.isPartial ? null : n.data.duration,
             offset,
             isRoot: n.id === spanId,

apps/webapp/app/v3/taskEventStore.server.ts

@@ -2,6 +2,7 @@
 import { Prisma, TaskEvent } from "@trigger.dev/database";
 import type { PrismaClient, PrismaReplicaClient } from "~/db.server";
 import { env } from "~/env.server";
+import { clampToEmergencySpanCap } from "~/v3/eventRepository/emergencySpanCap.server";
 
 export type CommonTaskEvent = Omit<TaskEvent, "id">;
 export type TraceEvent = Pick<
@@ -192,7 +193,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT)}
       `;
     } else {
       return await this.readReplica.$queryRaw<TraceEvent[]>`
@@ -220,7 +221,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_SUMMARY_VIEW_COUNT)}
       `;
     }
   }
@@ -270,7 +271,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT)}
       `;
     } else {
       return await this.readReplica.$queryRaw<DetailedTraceEvent[]>`
@@ -299,7 +300,7 @@ export class TaskEventStore {
               : Prisma.empty
           }
         ORDER BY "startTime" ASC
-        LIMIT ${env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT}
+        LIMIT ${clampToEmergencySpanCap(env.MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT)}
       `;
     }
   }