feat(pool): add pool.memoryLimit to recycle workers by RSS

e2e/memory-limit/fixtures/rstest.config.mts

@@ -0,0 +1,17 @@
+import { defineConfig } from '@rstest/core';
+
+export default defineConfig({
+  // `isolate: false` is the only mode where `memoryLimit` is consulted
+  // — workers must be reused for the recycle decision to matter.
+  isolate: false,
+  // `memoryLimit: 2` parses as 2 bytes (values > 1 are bytes; values in
+  // `(0, 1]` would be a fraction of total system memory). Every worker
+  // is over-limit the instant it reports its first RSS sample, so the
+  // pool must dispose it before reusing. The fixture then asserts each
+  // test file ran in a distinct process, proving the cap was honored
+  // end-to-end.
+  pool: {
+    maxWorkers: 1,
+    memoryLimit: 2,
+  },
+});

e2e/memory-limit/fixtures/test/f1.test.ts

@@ -0,0 +1,10 @@
+import { describe, expect, it } from '@rstest/core';
+import { recordPid } from './record-pid';
+
+recordPid('f1');
+
+describe('memory limit recycle — file 1', () => {
+  it('runs and records its pid', () => {
+    expect(true).toBe(true);
+  });
+});

e2e/memory-limit/fixtures/test/f2.test.ts

@@ -0,0 +1,10 @@
+import { describe, expect, it } from '@rstest/core';
+import { recordPid } from './record-pid';
+
+recordPid('f2');
+
+describe('memory limit recycle — file 2', () => {
+  it('runs and records its pid', () => {
+    expect(true).toBe(true);
+  });
+});

e2e/memory-limit/fixtures/test/f3.test.ts

@@ -0,0 +1,10 @@
+import { describe, expect, it } from '@rstest/core';
+import { recordPid } from './record-pid';
+
+recordPid('f3');
+
+describe('memory limit recycle — file 3', () => {
+  it('runs and records its pid', () => {
+    expect(true).toBe(true);
+  });
+});

e2e/memory-limit/fixtures/test/f4.test.ts

@@ -0,0 +1,10 @@
+import { describe, expect, it } from '@rstest/core';
+import { recordPid } from './record-pid';
+
+recordPid('f4');
+
+describe('memory limit recycle — file 4', () => {
+  it('runs and records its pid', () => {
+    expect(true).toBe(true);
+  });
+});

e2e/memory-limit/fixtures/test/record-pid.ts

@@ -0,0 +1,17 @@
+import { appendFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+/**
+ * Record this file's `process.pid` into a shared log. The driver reads
+ * the log after the fixture finishes and asserts unique pids — proving
+ * the worker was recycled between tasks (the configured `memoryLimit:
+ * 2` byte cap is blown on every task, so every runner is disposed and
+ * a fresh one spawned).
+ */
+export const PID_LOG_PATH =
+  process.env.RSTEST_MEMLIMIT_LOG ?? join(tmpdir(), 'rstest-memory-limit.log');
+
+export const recordPid = (fileTag: string): void => {
+  appendFileSync(PID_LOG_PATH, `${process.pid}\t${fileTag}\n`);
+};

e2e/memory-limit/index.test.ts

@@ -0,0 +1,50 @@
+import { existsSync, readFileSync, unlinkSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from '@rstest/core';
+import { runRstestCli } from '../scripts/';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+describe('pool.memoryLimit — heap-based worker recycle', () => {
+  it('recycles the worker when its RSS exceeds the cap', async ({
+    onTestFinished,
+  }) => {
+    const logPath = join(
+      tmpdir(),
+      `rstest-memory-limit-${process.pid}-${Date.now()}.log`,
+    );
+    onTestFinished(() => {
+      if (existsSync(logPath)) unlinkSync(logPath);
+    });
+
+    const { expectExecSuccess } = await runRstestCli({
+      command: 'rstest',
+      args: ['run'],
+      onTestFinished,
+      options: {
+        nodeOptions: {
+          cwd: join(__dirname, './fixtures'),
+          env: {
+            RSTEST_MEMLIMIT_LOG: logPath,
+          },
+        },
+      },
+    });
+    await expectExecSuccess();
+
+    // 4 files × `maxWorkers: 1` × `memoryLimit: 1` byte → every worker's
+    // first reported RSS is over-limit, so the pool must dispose it
+    // before the next task. Result: each file lands in a distinct
+    // process.pid. If the cap was ignored (regression), all 4 lines
+    // would share a pid and this assertion would catch it.
+    const lines = readFileSync(logPath, 'utf8')
+      .split('\n')
+      .filter((l) => l.trim().length > 0);
+    expect(lines.length).toBe(4);
+    const pids = new Set(lines.map((line) => line.split('\t')[0]));
+    expect(pids.size).toBe(4);
+  });
+});

packages/core/src/pool/index.ts

@@ -29,6 +29,7 @@ import {
 import type { TraceEvent } from '../utils/trace';
 import { isMemorySufficient } from '../utils/memory';
 import { createDefaultMemoryGate } from './memoryGate';
+import { parseMemoryLimit } from './parseMemoryLimit';
 import { Pool } from './pool';
 import type { PoolWorkerKind } from './types';
 
@@ -369,11 +370,18 @@ export const createPool = async ({
     throw `Invalid pool configuration: maxWorkers(${maxWorkers}) cannot be less than minWorkers(${minWorkers}).`;
   }
 
+  // `memoryLimit` only matters when runners are reused (`isolate: false`);
+  // otherwise workers are single-use, so the recycle check would be dead
+  // code. Parse to bytes here so the pool stores a single int.
+  const memoryLimitBytes =
+    isolate === false ? parseMemoryLimit(poolOptions.memoryLimit) : undefined;
+
   const pool = new Pool({
     workerEntry: resolve(__dirname, './worker.js'),
     isolate,
     maxWorkers,
     minWorkers,
+    memoryLimitBytes,
     execArgv: [
       ...(poolOptions?.execArgv ?? []),
       ...execArgv,

packages/core/src/pool/parseMemoryLimit.ts

@@ -0,0 +1,72 @@
+import os from 'node:os';
+
+/**
+ * Parse a user-supplied `pool.memoryLimit` value into a byte count.
+ *
+ * Returns `0` when:
+ * - input is `undefined`, `null`, or non-positive
+ * - the string cannot be interpreted (caller may want to log/warn
+ *   separately; here we just disable the cap rather than crashing CI)
+ *
+ * Accepted forms (mirrors Vitest's `VmOptions.memoryLimit`):
+ * - `number >= 1` — interpreted as bytes
+ * - `number` in `(0, 1]` — fraction of total system memory
+ * - `string` with a unit suffix — case-insensitive:
+ *   - `kb`/`k` (1e3), `kib` (2^10)
+ *   - `mb`/`m` (1e6), `mib` (2^20)
+ *   - `gb`/`g` (1e9), `gib` (2^30)
+ *   - `%` — fraction of total system memory (e.g. `"20%"`)
+ * - `string` without a unit suffix — parsed as a number (same rules as
+ *   the number form above)
+ */
+export const parseMemoryLimit = (
+  input: number | string | undefined,
+): number => {
+  if (input === undefined || input === null) return 0;
+
+  const totalMemory = os.totalmem();
+
+  if (typeof input === 'string') {
+    // Pull a trailing unit suffix off (e.g. "1.5GB" → "1.5" + "GB").
+    const match = input.trim().match(/^(-?\d+(?:\.\d+)?)\s*([a-z%]+)?$/i);
+    if (!match) return 0;
+    const numeric = Number.parseFloat(match[1]!);
+    if (!Number.isFinite(numeric) || numeric <= 0) return 0;
+    const unit = (match[2] ?? '').toLowerCase();
+
+    switch (unit) {
+      case '':
+        return fromNumber(numeric, totalMemory);
+      case '%':
+        return Math.floor((numeric / 100) * totalMemory);
+      case 'k':
+      case 'kb':
+        return Math.floor(numeric * 1e3);
+      case 'kib':
+        return Math.floor(numeric * 1024);
+      case 'm':
+      case 'mb':
+        return Math.floor(numeric * 1e6);
+      case 'mib':
+        return Math.floor(numeric * 1024 * 1024);
+      case 'g':
+      case 'gb':
+        return Math.floor(numeric * 1e9);
+      case 'gib':
+        return Math.floor(numeric * 1024 * 1024 * 1024);
+      default:
+        return 0;
+    }
+  }
+
+  if (typeof input !== 'number' || !Number.isFinite(input) || input <= 0) {
+    return 0;
+  }
+  return fromNumber(input, totalMemory);
+};
+
+const fromNumber = (value: number, totalMemory: number): number => {
+  // `(0, 1]` → fraction of system memory; `> 1` → bytes verbatim.
+  if (value <= 1) return Math.floor(value * totalMemory);
+  return Math.floor(value);
+};

packages/core/src/pool/pool.ts

@@ -113,7 +113,17 @@ export class Pool {
       const workerId = this.acquireWorkerId();
       const worker = createPoolWorker(task, this.options, workerId);
       gate?.attachWorker(worker);
-      const runner = new PoolRunner(worker, { workerId });
+      // `memoryLimitBytes` only matters when runners are reused
+      // (`isolate: false`). For `isolate: true` the runner is single-use
+      // so the cap is irrelevant and we omit it to keep `isUsable()`
+      // hot-path free of the (cheap) comparison.
+      const runner = new PoolRunner(worker, {
+        workerId,
+        memoryLimitBytes:
+          this.options.isolate === false
+            ? this.options.memoryLimitBytes
+            : undefined,
+      });
       this.activeRunners.add(runner);
       try {
         await runner.start();

packages/core/src/pool/poolRunner.ts

@@ -64,6 +64,17 @@ let nextTaskSeq = 0;
 
 type PoolRunnerOptions = {
   workerId: number;
+  /**
+   * Recycle this runner after it reports an RSS above this many bytes.
+   * Disabled when `0` or omitted. Only meaningful when the pool reuses
+   * runners (`isolate: false`); `isolate: true` workers are single-use.
+   *
+   * The worker reports its `process.memoryUsage().rss` at the end of
+   * each task. If a value exceeds the limit, `isUsable()` flips false
+   * and the pool's existing dispose path spawns a fresh runner instead
+   * of returning this one to the idle pool.
+   */
+  memoryLimitBytes?: number;
 };
 
 /**
@@ -99,8 +110,19 @@ export class PoolRunner {
    */
   private crashed = false;
 
+  /**
+   * Last RSS (bytes) reported by the worker. Updated on each
+   * `runFinished` / `collectFinished` response carrying a `memory`
+   * field. Compared against `memoryLimitBytes` in `isUsable()` so a
+   * bloated worker is recycled on next dispatch attempt instead of
+   * returning to the idle pool.
+   */
+  private lastReportedRssBytes = 0;
+  private readonly memoryLimitBytes: number;
+
   constructor(worker: PoolWorker, options: PoolRunnerOptions) {
     this.workerId = options.workerId;
+    this.memoryLimitBytes = Math.max(0, options.memoryLimitBytes ?? 0);
     this.worker = worker;
 
     this.handleMessage = this.handleMessage.bind(this);
@@ -113,7 +135,14 @@ export class PoolRunner {
   }
 
   isUsable(): boolean {
-    return this.state === 'STARTED' && !this.crashed;
+    if (this.state !== 'STARTED' || this.crashed) return false;
+    if (
+      this.memoryLimitBytes > 0 &&
+      this.lastReportedRssBytes > this.memoryLimitBytes
+    ) {
+      return false;
+    }
+    return true;
   }
 
   start(): Promise<void> {
@@ -325,9 +354,11 @@ export class PoolRunner {
         this.startDeferred = undefined;
         return;
       case 'runFinished':
+        if (response.memory) this.lastReportedRssBytes = response.memory.rss;
         this.resolveTask('run', response.taskId, response.result);
         return;
       case 'collectFinished':
+        if (response.memory) this.lastReportedRssBytes = response.memory.rss;
         this.resolveTask('collect', response.taskId, response.result);
         return;
       case 'fatal_error': {

packages/core/src/pool/types.ts

@@ -15,6 +15,12 @@ export type PoolOptions = {
   maxWorkers: number;
   minWorkers: number;
   isolate: boolean;
+  /**
+   * Recycle a reused runner once its last-reported RSS exceeds this
+   * many bytes. Disabled when omitted or `0`. See
+   * `RstestPoolOptions.memoryLimit` for the user-facing knob.
+   */
+  memoryLimitBytes?: number;
   env?: Record<string, string>;
   execArgv?: string[];
   /**

packages/core/src/types/config.ts

@@ -25,6 +25,28 @@ export type RstestPoolOptions = {
   minWorkers?: number | string;
   /** Pass additional arguments to node process in the child processes. */
   execArgv?: string[];
+  /**
+   * Recycle a reused worker once its RSS exceeds this threshold. Only
+   * meaningful when `isolate: false` (workers are reused across files);
+   * a no-op under the default `isolate: true` since workers are
+   * single-use anyway.
+   *
+   * Accepts:
+   * - `number` >= 1 — bytes (e.g. `1_500_000_000` for 1.5 GB)
+   * - `number` in `(0, 1]` — fraction of total system memory
+   * - `string` with a unit suffix — `"512MB"`, `"1.5GB"`, `"20%"`,
+   *   `"1GiB"`, … (`%` is fraction of total system memory)
+   *
+   * After each test file the worker reports `process.memoryUsage().rss`
+   * to the host. When that exceeds the parsed limit, the pool disposes
+   * the worker and spawns a fresh one. Heap accumulated from prior
+   * files (vendor modules, JSDOM nodes, React fiber trees) is
+   * reclaimed by process exit.
+   *
+   * Useful for long suites under `isolate: false` where a single
+   * worker's heap can grow into GC-thrash territory.
+   */
+  memoryLimit?: number | string;
 };
 
 export type BundleDependencyPattern = string | RegExp;