feat: determine where OOM came from

Author: doc-han

packages/engine-multi/src/errors.ts

@@ -71,14 +71,21 @@ export class AutoinstallError extends EngineError {
   }
 }
 
+// Where an OOM kill came from:
+// - 'heap'   the V8 heap limit (thread resourceLimits or --max-old-space-size)
+// - 'cgroup' the OS cgroup memory.max ceiling (kernel SIGKILL)
+export type OOMSource = 'heap' | 'cgroup';
+
 export class OOMError extends EngineError {
   severity = 'kill';
   name = 'OOMError';
   message;
+  source: OOMSource;
 
-  constructor() {
+  constructor(source: OOMSource = 'heap') {
     super();
 
+    this.source = source;
     this.message = `Run exceeded maximum memory usage`;
   }
 }

packages/engine-multi/src/util/serialize-error.ts

@@ -4,6 +4,7 @@ export default (error: any) => {
     name: error.name,
     type: error.type,
     subtype: error.subtype,
+    source: error.source,
     severity: error.severity || 'crash',
   };
 };

packages/engine-multi/src/worker/pool.ts

@@ -197,10 +197,14 @@ function createPool(script: string, options: PoolOptions = {}, logger: Logger) {
           // the cgroup's OOM counter before falling back to scraping stderr.
           const handle = worker.pid ? cgroups[worker.pid] : null;
           if (handle && hasOomKill(handle)) {
+            logger.error(
+              `pool: worker ${worker.pid} was killed by the OS for exceeding ` +
+                `its cgroup memory limit (${options.cgroupMemoryLimitMb}mb)`
+            );
             killWorker(worker);
             // restore a placeholder to the queue
             finish(false);
-            reject(new OOMError());
+            reject(new OOMError('cgroup'));
             return;
           }
 
@@ -214,6 +218,9 @@ function createPool(script: string, options: PoolOptions = {}, logger: Logger) {
             if (worker.stderr && worker.stderr?.readableLength > 0) {
               for await (const line of rl) {
                 if (line.match(/JavaScript heap out of memory/)) {
+                  logger.error(
+                    `pool: worker ${worker.pid} exceeded the V8 heap limit`
+                  );
                   killWorker(worker);
                   // restore a placeholder to the queue
                   finish(false);
@@ -301,6 +308,8 @@ function createPool(script: string, options: PoolOptions = {}, logger: Logger) {
             // @ts-ignore
             e.severity = evt.error.severity;
             e.name = evt.error.name;
+            // @ts-ignore preserve the OOM source ('heap'/'cgroup') across IPC
+            e.source = evt.error.source;
             reject(e);
 
             finish(worker);

packages/engine-multi/test/worker/cgroup-enforcement.test.ts

@@ -28,9 +28,10 @@ linuxOnly(
   async (t) => {
     const pool = createPool(workerPath, { cgroupMemoryLimitMb }, logger);
 
-    await t.throwsAsync(() => pool.exec('blowNativeMemory', []), {
+    const err = await t.throwsAsync(() => pool.exec('blowNativeMemory', []), {
       name: 'OOMError',
     });
+    t.is((err as any).source, 'cgroup');
 
     await pool.destroy();
   }

packages/engine-multi/test/worker/pool.test.ts

@@ -179,9 +179,11 @@ test('throw if memory limit is exceeded', async (t) => {
 
   try {
     await pool.exec('blowMemory', [], { memoryLimitMb: 100 });
+    t.fail('expected the run to OOM');
   } catch (e: any) {
     t.is(e.message, 'Run exceeded maximum memory usage');
     t.is(e.name, 'OOMError');
+    t.is(e.source, 'heap');
   }
 });
 
@@ -200,9 +202,13 @@ test('child process should not have --max-old-space-size when memoryLimitMb is n
 test('pool recovers after process-level OOM', async (t) => {
   const pool = createPool(workerPath, { memoryLimitMb: 50 }, logger);
 
-  await t.throwsAsync(() => pool.exec('blowMemory', [], { memoryLimitMb: 20 }), {
-    name: 'OOMError',
-  });
+  const err = await t.throwsAsync(
+    () => pool.exec('blowMemory', [], { memoryLimitMb: 20 }),
+    {
+      name: 'OOMError',
+    }
+  );
+  t.is((err as any).source, 'heap');
 
   // Pool should still be functional after the OOM
   const result = await pool.exec('test', [42]);