From 596a9bb90ce9723eec60308c238c1021051cc9f1 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Sun, 24 May 2026 16:21:15 +0100 Subject: [PATCH] fix(webapp): retain sessions-replication singleton import via globalThis assignment (#3738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linear: [TRI-9864](https://linear.app/triggerdotdev/issue/TRI-9864) (Urgent) Production incident: [TRI-9863](https://linear.app/triggerdotdev/issue/TRI-9863) (mitigated by image revert in cloud#910) ## Bug `apps/webapp/package.json` declares `"sideEffects": false`. PR #3333 (`71d98b4e`) replaced the previous real method-call retention idiom at the two `sessionsReplicationInstance` import sites with: ```ts import { sessionsReplicationInstance } from "..."; void sessionsReplicationInstance; ``` esbuild treats `void ;` as a pure expression statement under `sideEffects: false` and **tree-shakes the entire import** — including the `singleton(...)` call inside `sessionsReplicationInstance.server.ts` which is the only thing that fires `initializeSessionsReplicationInstance()`. The sessions→ClickHouse logical replication worker never starts, the slot is unconsumed, lag grows. ### How it manifested in production cloud#907's image bump rolled the `SessionReplicationService` ECS task on prod at 14:32 UTC. The new container's startup log emitted `🗃️ Runs replication service enabled` but **not** `🗃️ Sessions replication service enabled` or `🗃️ Sessions replication service started`. CloudWatch `OldestReplicationSlotLag` grew at ~220 MB/min and the `High replication lag` alarm fired at 14:37 UTC. Prod was reverted to the previous image (cloud#910) to stop the bleed. ### Verification `grep` of the built bundle `apps/webapp/build/index.js` (built from `c0365d36`): - **3** occurrences of `Runs replication` / `runsReplicationInstance` strings ✅ - **0** occurrences of `Sessions replication` / `sessionsReplicationInstance` / `SessionsReplicationService` ❌ The runs path survives tree-shaking because `adminWorker.server.ts` and `admin.api.v1.runs-replication.*` routes have real method calls (`.start()`, `.teardown()`, `.backfill()`) — observable uses the tree-shaker must preserve. The sessions singleton has no real callers, only the `void` no-ops, hence its complete elimination from the bundle. ## Fix Replace `void sessionsReplicationInstance;` with an assignment to `globalThis`, an unambiguous observable side effect the bundler cannot eliminate: ```ts (globalThis as Record).__sessionsReplicationInstance = sessionsReplicationInstance; ``` Applied at both call sites: `apps/webapp/app/entry.server.tsx` and `apps/webapp/app/v3/services/adminWorker.server.ts`. Surrounding comments updated to document the bundler interaction so the next maintainer doesn't reintroduce `void`. ## Out of scope (follow-ups) - **Robustness improvement**: change `apps/webapp/package.json` from `"sideEffects": false` to an allowlist that includes `*Instance.server.ts` files. Prevents the same regression shape via any future `*Instance` singleton. - **Build-time check**: add a `grep` post-build step in `publish.yml` requiring `"Sessions replication"` to appear in `apps/webapp/build/index.js`. Catches this exact regression at CI time. ## Test plan - [x] `pnpm run typecheck --filter webapp` clean - [ ] After merge + publish: confirm new image's `SessionReplicationService` container logs `🗃️ Sessions replication service enabled` and `🗃️ Sessions replication service started` at startup - [ ] After re-deploying to prod: confirm `OldestReplicationSlotLag` stops growing and drains --- apps/webapp/app/entry.server.tsx | 11 ++++++++++- apps/webapp/app/v3/services/adminWorker.server.ts | 12 +++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index db72b0364c2..60c234402d5 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -30,8 +30,17 @@ import { // on webapp startup. The singleton's initializer wires start (gated on // `clickhouseFactory.isReady()`) and SIGTERM/SIGINT shutdown — mirrors // runsReplicationInstance. +// +// IMPORTANT: do NOT replace this with `void sessionsReplicationInstance;`. +// `apps/webapp/package.json` declares `"sideEffects": false`, so esbuild +// treats `void ;` as a pure expression statement and tree-shakes +// the entire import — the singleton's initializer never fires and the +// sessions→ClickHouse logical replication slot stops being consumed. Assigning +// to globalThis is an unambiguous side effect the bundler must preserve. See +// TRI-9864 for the incident write-up. import { sessionsReplicationInstance } from "./services/sessionsReplicationInstance.server"; -void sessionsReplicationInstance; +(globalThis as Record).__sessionsReplicationInstance = + sessionsReplicationInstance; const ABORT_DELAY = 30000; diff --git a/apps/webapp/app/v3/services/adminWorker.server.ts b/apps/webapp/app/v3/services/adminWorker.server.ts index 2e4d1b066cb..cf3dbd57c6c 100644 --- a/apps/webapp/app/v3/services/adminWorker.server.ts +++ b/apps/webapp/app/v3/services/adminWorker.server.ts @@ -6,10 +6,16 @@ import { logger } from "~/services/logger.server"; import { runsReplicationInstance } from "~/services/runsReplicationInstance.server"; // Reference-hold the sessions-replication singleton so module evaluation runs // its initializer (creates the ClickHouse client, subscribes to the logical -// replication slot, wires signal handlers) when the webapp boots. A bare -// side-effect import gets tree-shaken by the bundler. +// replication slot, wires signal handlers) when the webapp boots. +// +// IMPORTANT: do NOT replace with `void sessionsReplicationInstance;`. With +// `"sideEffects": false` in apps/webapp/package.json, esbuild treats `void X;` +// as a pure expression statement and eliminates the import — the singleton +// initializer never fires. Assignment to globalThis is an observable side +// effect the bundler must preserve. See TRI-9864. import { sessionsReplicationInstance } from "~/services/sessionsReplicationInstance.server"; -void sessionsReplicationInstance; +(globalThis as Record).__sessionsReplicationInstance = + sessionsReplicationInstance; import { singleton } from "~/utils/singleton"; import { tracer } from "../tracer.server"; import { $replica } from "~/db.server";