Fix stale page module cache poisoning

packages/realm-server/prerender/page-pool.ts

@@ -359,6 +359,12 @@ export class PagePool {
   // throughout this file so a single source of truth flows into
   // both PagePool's own logs and the capture module's logs.
   #affinityKeyByPageId = new Map<string, string>();
+  // Test-only: per-pageId set of module URLs that should fail to render
+  // on that page, modeling a pool page whose loaded host bundle is stale
+  // (predates an export the module imports). Populated only via
+  // `__test_poisonPage`; the module render path consults it through
+  // `__test_isPagePoisonedForModule`. Production never writes to it.
+  #poisonedModulesByPageId = new Map<string, Set<string>>();
   // Fired from `disposeAffinity` after an affinity's tabs are torn down.
   // Consumed by the Prerenderer to clear `clearCache` batch ownership
   // for the affinity (CS-10758 step 3) — stale ownership across a page
@@ -889,7 +895,7 @@ export class PagePool {
   async getPage(
     affinityKey: string,
     queue: PrerenderQueue = 'file',
-    opts?: { signal?: AbortSignal; priority?: number },
+    opts?: { signal?: AbortSignal; priority?: number; freshPage?: boolean },
   ): Promise<{
     page: Page;
     reused: boolean;
@@ -980,6 +986,7 @@ export class PagePool {
           queue,
           signal,
           priority,
+          opts?.freshPage === true,
         ));
     } catch (e) {
       releaseAdmission?.();
@@ -1890,6 +1897,7 @@ export class PagePool {
     queue: PrerenderQueue,
     signal?: AbortSignal,
     priority: number = 0,
+    freshPage: boolean = false,
   ): Promise<{
     entry: PoolEntry;
     reused: boolean;
@@ -1921,6 +1929,45 @@ export class PagePool {
     let entryList = entries
       ? [...entries].filter((entry) => !entry.closing)
       : [];
+    // Error-cache revalidation: the affinity's idle warm tabs may be on a
+    // stale host bundle. Retire them so `#selectLRUTab` can't keep handing
+    // a stale tab to later renders (which would poison other modules in
+    // the realm), then serve this render on a fresh standby. Only idle
+    // tabs are retired — in-flight renders on busy tabs are left to
+    // finish, so unlike disposing the whole affinity this never tears down
+    // active work. Falls through to the normal selection below when no
+    // standby is free, so a render is never starved.
+    if (freshPage) {
+      this.#retireIdleAffinityTabs(affinityKey);
+      // Recompute: the tabs just marked `closing` must not be eligible for
+      // the fall-through selection below.
+      entries = this.#affinityPages.get(affinityKey);
+      entryList = entries ? [...entries].filter((entry) => !entry.closing) : [];
+      let standby = this.#commandeerDormantTab(affinityKey, {
+        standbyOnly: true,
+      });
+      if (!standby) {
+        // No standby free this instant — await the standby machinery
+        // before falling back to reuse. This is UNCONDITIONAL (not gated
+        // on `currentStandbyCount() < desiredStandbyCount()`) for the same
+        // reason the deadlock-escape path below is: a standby mid-creation
+        // inflates `currentStandbyCount` (= size + creating) to meet
+        // `desired` while `#standbys.size` is still 0, so a count guard
+        // would skip the await and fall through to reusing the stale tab.
+        // `#ensureStandbyPool` returns the in-flight refill if one is
+        // creating, and is a no-op when the pool genuinely has no room.
+        let startedAt = Date.now();
+        await this.#ensureStandbyPool();
+        tabStartupMs += Date.now() - startedAt;
+        standby = this.#commandeerDormantTab(affinityKey, {
+          standbyOnly: true,
+        });
+      }
+      if (standby) {
+        let releaseTab = await standby.queue.acquire(signal, priority);
+        return { entry: standby, reused: false, releaseTab, tabStartupMs };
+      }
+    }
     let idle = entryList.filter((entry) => entry.queue.pendingCount === 0);
     if (idle.length > 0) {
       let entry = this.#selectLRUTab(idle);
@@ -2333,6 +2380,41 @@ export class PagePool {
     return this.#reassignAffinityTab(chosen, affinityKey);
   }
 
+  // Close an affinity's IDLE tabs (pendingCount 0, not closing /
+  // transitioning) and drop them from `#affinityPages`, leaving any
+  // in-flight tabs to finish. Used by the `freshPage` error-cache-
+  // revalidation path to retire a possibly-stale warm tab so
+  // `#selectLRUTab` can't re-hand it to a later render. Mirrors the
+  // per-entry, non-awaited close in `disposeAffinity`; `#closeEntry`
+  // retains the shared BrowserContext as an orphan (same as a normal tab
+  // close), so a subsequent spawn can still reuse the warm loader.
+  #retireIdleAffinityTabs(affinityKey: string): void {
+    let entries = this.#affinityPages.get(affinityKey);
+    if (!entries) {
+      return;
+    }
+    for (let entry of [...entries]) {
+      if (entry.closing || entry.transitioning) {
+        continue;
+      }
+      if (entry.queue.pendingCount !== 0) {
+        continue;
+      }
+      entry.closing = true;
+      let p = this.#closeEntry(entry).finally(() => {
+        let current = this.#affinityPages.get(affinityKey);
+        if (!current) {
+          return;
+        }
+        current.delete(entry);
+        if (current.size === 0) {
+          this.#affinityPages.delete(affinityKey);
+        }
+      });
+      void p;
+    }
+  }
+
   #assignStandbyToAffinity(
     standby: StandbyEntry,
     affinityKey: string,
@@ -2849,6 +2931,47 @@ export class PagePool {
     this.#recordRevokedException(pageId, exceptionId);
   }
 
+  // Test-only seam: mark a specific pool page as unable to render a
+  // module, modeling a page running a stale host bundle. A real stale
+  // page differs from its peers only by the bytes already loaded into
+  // its browser context — something a test can't reproduce without
+  // shipping two host builds — so the differentiation is injected here
+  // by pageId. The module render path checks this via
+  // `__test_isPagePoisonedForModule` and synthesizes the same
+  // module-error a genuine in-page failure would. Production never
+  // calls this; the registry is otherwise always empty.
+  __test_poisonPage(pageId: string, moduleURL: string): void {
+    let bucket = this.#poisonedModulesByPageId.get(pageId);
+    if (!bucket) {
+      bucket = new Set();
+      this.#poisonedModulesByPageId.set(pageId, bucket);
+    }
+    bucket.add(this.#normalizePoisonKey(moduleURL));
+  }
+
+  // Test-only seam: clear all injected page poison. Lets a test flip a
+  // page back to "healthy" to exercise the recovery path.
+  __test_clearPoisonedPages(): void {
+    this.#poisonedModulesByPageId.clear();
+  }
+
+  // Consulted by the module render path. Returns true when a prior
+  // `__test_poisonPage` marked this page+module as stale. Always false
+  // in production (the registry is never populated).
+  __test_isPagePoisonedForModule(pageId: string, moduleURL: string): boolean {
+    let bucket = this.#poisonedModulesByPageId.get(pageId);
+    if (!bucket) {
+      return false;
+    }
+    return bucket.has(this.#normalizePoisonKey(moduleURL));
+  }
+
+  // Drop any executable extension so the poison key matches whether the
+  // caller passes `…/person.gts` or the extensionless module URL.
+  #normalizePoisonKey(moduleURL: string): string {
+    return moduleURL.replace(/\.(gts|gjs|ts|js)$/, '');
+  }
+
   #attachPageConsole(page: Page, affinityKey: string, pageId: string): void {
     page.on('console', async (message: ConsoleMessage) => {
       try {

packages/realm-server/prerender/prerender-app.ts

@@ -112,6 +112,9 @@ export function buildPrerenderApp(options: {
     // affinity admission semaphore / global render semaphore for
     // priority-aware dequeue.
     priority?: number;
+    // Error-cache revalidation hint — see ModulePrerenderArgs.freshPage.
+    // Honored by prerenderModule; ignored by visit/command routes.
+    freshPage?: boolean;
   };
 
   type PrerenderArgs = RouteBaseArgs & {
@@ -203,6 +206,7 @@ export function buildPrerenderApp(options: {
       { value: rawAffinityValue, name: 'affinityValue' },
     ]);
     let priority = parsePriority(attrs);
+    let freshPage = attrs?.freshPage === true;
     return {
       args:
         missing.length > 0
@@ -215,6 +219,7 @@ export function buildPrerenderApp(options: {
               auth: rawAuth as string,
               renderOptions,
               ...(priority !== undefined ? { priority } : {}),
+              ...(freshPage ? { freshPage: true } : {}),
             },
       missing,
       missingMessage:

packages/realm-server/prerender/prerenderer.ts

@@ -173,6 +173,16 @@ export class Prerenderer {
     this.#pagePool.__test_seedRevokedException(pageId, entry, exceptionId);
   }
 
+  // Test-only seam — see PagePool.__test_poisonPage.
+  __test_poisonPage(pageId: string, moduleURL: string): void {
+    this.#pagePool.__test_poisonPage(pageId, moduleURL);
+  }
+
+  // Test-only seam — see PagePool.__test_clearPoisonedPages.
+  __test_clearPoisonedPages(): void {
+    this.#pagePool.__test_clearPoisonedPages();
+  }
+
   async stop(): Promise<void> {
     if (this.#cleanupInterval) {
       clearInterval(this.#cleanupInterval);
@@ -309,6 +319,7 @@ export class Prerenderer {
     renderOptions,
     priority,
     signal,
+    freshPage,
   }: {
     affinityType: AffinityType;
     affinityValue: string;
@@ -323,6 +334,8 @@ export class Prerenderer {
     // global render semaphore for priority-aware dequeue.
     priority?: number;
     signal?: AbortSignal;
+    // Error-cache revalidation hint — see ModulePrerenderArgs.freshPage.
+    freshPage?: boolean;
   }): Promise<{
     response: ModuleRenderResponse;
     timings: Timings;
@@ -380,6 +393,7 @@ export class Prerenderer {
             renderOptions: attemptOptions,
             priority,
             signal,
+            freshPage,
             onTabAcquired: activity.markRunning,
           });
         } catch (e) {

packages/realm-server/prerender/remote-prerenderer.ts

@@ -212,7 +212,14 @@ export function createRemotePrerenderer(
   }
 
   return {
-    async prerenderModule({ realm, url, auth, renderOptions, priority }) {
+    async prerenderModule({
+      realm,
+      url,
+      auth,
+      renderOptions,
+      priority,
+      freshPage,
+    }) {
       return await requestWithRetry<ModuleRenderResponse>(
         'prerender-module',
         'prerender-module-request',
@@ -224,6 +231,7 @@ export function createRemotePrerenderer(
           auth,
           renderOptions: renderOptions ?? {},
           ...(priority !== undefined ? { priority } : {}),
+          ...(freshPage ? { freshPage: true } : {}),
         },
       );
     },

packages/realm-server/prerender/render-runner.ts

@@ -143,6 +143,7 @@ export class RenderRunner {
     queue: PrerenderQueue,
     signal?: AbortSignal,
     priority?: number,
+    freshPage?: boolean,
   ) {
     let lastAuth = this.#lastAuthByAffinity.get(affinityKey);
     if (lastAuth) {
@@ -161,6 +162,7 @@ export class RenderRunner {
     let pageInfo = await this.#pagePool.getPage(affinityKey, queue, {
       signal,
       ...(priority !== undefined ? { priority } : {}),
+      ...(freshPage ? { freshPage: true } : {}),
     });
     this.#lastAuthByAffinity.set(affinityKey, auth);
     return pageInfo;
@@ -541,6 +543,7 @@ export class RenderRunner {
     priority,
     signal,
     onTabAcquired,
+    freshPage,
   }: {
     affinityType: AffinityType;
     affinityValue: string;
@@ -551,6 +554,10 @@ export class RenderRunner {
     renderOptions?: RenderRouteOptions;
     priority?: number;
     signal?: AbortSignal;
+    // Error-cache revalidation hint — see ModulePrerenderArgs.freshPage.
+    // Routes this render onto a fresh page instead of the affinity's warm
+    // (possibly stale-bundle) tab.
+    freshPage?: boolean;
     // Fires after `getPageForAffinity` resolves AND the per-page
     // console/exception bucket has been reset — i.e. when this attempt
     // has a page AND the bucket is empty for THIS render. The reset
@@ -581,6 +588,7 @@ export class RenderRunner {
         'module',
         signal,
         priority,
+        freshPage,
       );
     const poolInfo: PoolInfo = {
       pageId: pageId ?? 'unknown',
@@ -602,6 +610,39 @@ export class RenderRunner {
       // inside the try so `finally { release() }` frees the tab slot
       // if the caller aborted during the getPage handoff.
       throwIfAborted(signal, 'queued');
+
+      // Test-only: a page marked stale by `__test_poisonPage` renders
+      // the module exactly as a page on an outdated host bundle would —
+      // the import of an export the bundle predates throws, surfacing as
+      // a module-error. Synthesize that here so the affinity-routing
+      // behavior around a stale page can be exercised without shipping a
+      // second host build. The registry is always empty in production.
+      if (this.#pagePool.__test_isPagePoisonedForModule(pageId, url)) {
+        let renderError = buildInvalidModuleResponseError(
+          page,
+          `Module '@cardstack/runtime-common' has no exported member 'buildWaiter'.`,
+          { title: 'Module Error', evict: false },
+        );
+        renderError.type = 'module-error';
+        let response: ModuleRenderResponse = {
+          id: url,
+          status: 'error',
+          nonce: String(this.#nonce),
+          isShimmed: false,
+          lastModified: 0,
+          createdAt: 0,
+          deps: renderError.error.deps ?? [],
+          definitions: {},
+          error: renderError,
+        };
+        response.error = this.#mergeConsoleErrors(pageId, response.error);
+        return {
+          response,
+          timings: { launchMs, renderMs: 0, waits },
+          pool: poolInfo,
+        };
+      }
+
       await page.evaluate((sessionAuth) => {
         localStorage.setItem('boxel-session', sessionAuth);
       }, auth);

packages/realm-server/tests/index.ts

@@ -183,11 +183,13 @@ const ALL_TEST_FILES: string[] = [
   './indexing-test',
   './lazy-mount-test',
   './listener-dispatcher-test',
+  './module-cache-poison-test',
   './module-cache-race-test',
   './module-syntax-test',
   './network-inflight-tracker-test',
   './permissions/permission-checker-test',
   './prerendering-test',
+  './prerender-stale-page-affinity-test',
   './prerender-server-test',
   './prerender-manager-test',
   './prerender-affinity-activity-test',