test_runner: support runner bailout at files level

Author: pmarchini

lib/internal/test_runner/reporter/spec.js

@@ -78,6 +78,8 @@ class SpecReporter extends Transform {
   }
   #handleEvent({ type, data }) {
     switch (type) {
+      case 'test:bail':
+        return `${reporterColorMap['test:bail']}Bailing out! no new test files will be started!${colors.white}\n`;
       case 'test:fail':
         if (data.details?.error?.failureType !== kSubtestsFailed) {
           ArrayPrototypePush(this.#failedTests, data);

lib/internal/test_runner/reporter/utils.js

@@ -37,6 +37,9 @@ const reporterColorMap = {
   get 'test:diagnostic'() {
     return colors.blue;
   },
+  get 'test:bail'() {
+    return colors.yellow;
+  },
   get 'info'() {
     return colors.blue;
   },

lib/internal/test_runner/runner.js

@@ -257,16 +257,6 @@ class FileTest extends Test {
       item.data.details.error = deserializeError(item.data.details.error);
     }
 
-    if (item.type === 'test:fail' && this.root.harness.bail && !this.root.harness.bailedOut) {
-      // Trigger bail if enabled and this is the first failure
-      this.root.harness.bailedOut = true;
-      this.reporter[kEmitMessage]('test:bail', {
-        __proto__: null,
-        file: this.name,
-        test: item.data,
-      });
-    }
-
     if (item.type === 'test:pass' || item.type === 'test:fail') {
       item.data.testNumber = isTopLevel ? (this.root.harness.counters.topLevel + 1) : item.data.testNumber;
       countCompletedTest({
@@ -776,15 +766,28 @@ function run(options = kEmptyObject) {
     if (watch) {
       throw new ERR_INVALID_ARG_VALUE('options.bail', watch, 'bail not supported with watch mode');
     }
-    if (isolation === 'none') {
-      throw new ERR_INVALID_ARG_VALUE('options.bail', isolation, 'bail not supported with \'none\' isolation');
-    }
   }
 
   let teardown;
   let postRun;
   let filesWatcher;
   let runFiles;
+
+  if (bail) {
+    root.reporter.on('test:fail', (item) => {
+      if (root.harness.bail && !root.harness.bailedOut) {
+        root.harness.bailedOut = true;
+        queueMicrotask(() => {
+          root.reporter[kEmitMessage]('test:bail', {
+            __proto__: null,
+            file: item.name,
+            test: item.data,
+          });
+        });
+      }
+    });
+  }
+
   const opts = {
     __proto__: null,
     root,
@@ -854,7 +857,6 @@ function run(options = kEmptyObject) {
           }
         }
       };
-
     } else {
       runFiles = () => {
         root.harness.bootstrapPromise = null;
@@ -878,17 +880,21 @@ function run(options = kEmptyObject) {
         return subtest;
       };
     } else {
+
       runFiles = async () => {
-        const { promise, resolve: finishBootstrap } = PromiseWithResolvers();
+        // Ensure global bootstrap is completed before running files, then allow
+        // subtests to start immediately so bail can stop further file imports.
+        if (root.harness.bootstrapPromise) {
+          await root.harness.bootstrapPromise;
+          root.harness.bootstrapPromise = null;
+        }
+        root.harness.buildPromise = null;
 
         await root.runInAsyncScope(async () => {
           const parentURL = pathToFileURL(cwd + sep).href;
           const cascadedLoader = esmLoader.getOrInitializeCascadedLoader();
           let topLevelTestCount = 0;
-
-          root.harness.bootstrapPromise = root.harness.bootstrapPromise ?
-            SafePromiseAllReturnVoid([root.harness.bootstrapPromise, promise]) :
-            promise;
+          let failedCount = root.harness.counters.failed;
 
           // We need to setup the user modules in the test runner if we are running with
           // --test-isolation=none and --test in order to avoid loading the user modules
@@ -906,7 +912,12 @@ function run(options = kEmptyObject) {
           }
 
           for (let i = 0; i < testFiles.length; ++i) {
+            if (root.harness.bail && root.harness.bailedOut) {
+              break;
+            }
+
             const testFile = testFiles[i];
+            const testFilePath = resolve(cwd, testFile);
             const fileURL = pathToFileURL(resolve(cwd, testFile));
             const parent = i === 0 ? undefined : parentURL;
             let threw = false;
@@ -933,22 +944,54 @@ function run(options = kEmptyObject) {
               if (threw) {
                 subtest.fail(importError);
               }
-              startSubtestAfterBootstrap(subtest);
+            startSubtestAfterBootstrap(subtest);
             }
 
             topLevelTestCount = root.subtests.length;
+
+            // Run pending subtests created so far; if bail triggers, stop loading more files.
+            await root.processPendingSubtests();
+            // Allow reporter events (including bail) to flush before continuing.
+            await new Promise((resolve) => setImmediate(resolve));
+            // Ensure any subtest chains spawned by this file are finished.
+            if (root.subtestsPromise?.promise) {
+              await root.subtestsPromise.promise;
+            }
+
+            if (root.harness.bail) {
+              if (root.harness.counters.failed > failedCount) {
+                root.harness.bailedOut = true;
+              } else {
+                const fileFailed = ArrayPrototypeSome(
+                  root.subtests,
+                  (t) => t.loc?.file === testFilePath && t.error != null,
+                );
+                if (fileFailed) {
+                  root.harness.bailedOut = true;
+                }
+              }
+            }
+
+            if (root.harness.bail && root.harness.bailedOut) {
+              break;
+            }
+
+            failedCount = root.harness.counters.failed;
           }
         });
 
         debug('beginning test execution');
         root.entryFile = null;
-        finishBootstrap();
+        if (root.harness.bail && root.harness.bailedOut) {
+          return;
+        }
         return root.processPendingSubtests();
       };
     }
   }
 
   const runChain = async () => {
+
     if (root.harness?.bootstrapPromise) {
       await root.harness.bootstrapPromise;
     }

lib/internal/test_runner/test.js

@@ -953,6 +953,13 @@ class Test extends AsyncResource {
       this.passed = false;
     }
 
+    // Immediate bail signalling for isolation='none' path: mark harness so the
+    // runner can stop loading additional files as soon as a top-level test
+    // fails. This is a no-op when bail is disabled.
+    if (this.harness?.bail) {
+      this.harness.bailedOut = true;
+    }
+
     this.error = err;
   }
 

test/fixtures/test-runner/bail/bail-test-2-fail.js

@@ -7,5 +7,5 @@ test('failing test 1', () => {
 });
 
 test('failing test 2', () => {
-  assert.strictEqual(3, 4, 'This test should also fail but might not run');
+  assert.strictEqual(3, 4, 'This test fails as well');
 });

test/fixtures/test-runner/output/bail_concurrency_1.js

@@ -0,0 +1,13 @@
+'use strict';
+const fixtures = require('../../../common/fixtures');
+const { spec } = require('node:test/reporters');
+const { run } = require('node:test');
+
+const files = [
+  fixtures.path('test-runner', 'bail', 'bail-test-1-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-2-fail.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-3-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-4-pass.js'),
+];
+
+run({ bail: true, concurrency: 1, files }).compose(spec).compose(process.stdout);

test/fixtures/test-runner/output/bail_concurrency_1.snapshot

@@ -0,0 +1,55 @@
+✔ test 1 passes (*ms)
+✔ test 2 passes (*ms)
+✖ failing test 1 (*ms)
+Bailing out! no new test files will be started!
+✖ failing test 2 (*ms)
+ℹ tests 4
+ℹ suites 0
+ℹ pass 2
+ℹ fail 2
+ℹ cancelled 0
+ℹ skipped 0
+ℹ todo 0
+ℹ duration_ms *
+
+✖ failing tests:
+
+*
+✖ failing test 1 (*ms)
+  AssertionError [ERR_ASSERTION]: This test should fail
+  
+  1 !== 2
+  
+      *
+      *
+      *
+      *
+      * {
+    generatedMessage: false,
+    code: 'ERR_ASSERTION',
+    actual: 1,
+    expected: 2,
+    operator: 'strictEqual',
+    diff: 'simple'
+  }
+
+*
+✖ failing test 2 (*ms)
+  AssertionError [ERR_ASSERTION]: This test fails as well
+  
+  3 !== 4
+  
+      *
+      *
+      *
+      *
+      *
+      *
+      * {
+    generatedMessage: false,
+    code: 'ERR_ASSERTION',
+    actual: 3,
+    expected: 4,
+    operator: 'strictEqual',
+    diff: 'simple'
+  }

test/fixtures/test-runner/output/bail_concurrency_1_isolation_none.js

@@ -0,0 +1,13 @@
+'use strict';
+const fixtures = require('../../../common/fixtures');
+const { spec } = require('node:test/reporters');
+const { run } = require('node:test');
+
+const files = [
+  fixtures.path('test-runner', 'bail', 'bail-test-1-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-2-fail.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-3-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-4-pass.js'),
+];
+
+run({ bail: true, concurrency: 1, isolation: 'none', files }).compose(spec).compose(process.stdout);

test/fixtures/test-runner/output/bail_concurrency_1_isolation_none.snapshot

@@ -0,0 +1,60 @@
+✔ test 1 passes (*ms)
+✔ test 2 passes (*ms)
+✖ failing test 1 (*ms)
+Bailing out! no new test files will be started!
+✖ failing test 2 (*ms)
+ℹ tests 4
+ℹ suites 0
+ℹ pass 2
+ℹ fail 2
+ℹ cancelled 0
+ℹ skipped 0
+ℹ todo 0
+ℹ duration_ms *
+
+✖ failing tests:
+
+*
+✖ failing test 1 (*ms)
+  AssertionError [ERR_ASSERTION]: This test should fail
+  
+  1 !== 2
+  
+      *
+      *
+      *
+      *
+      *
+      *
+      *
+      *
+      *
+      * {
+    generatedMessage: false,
+    code: 'ERR_ASSERTION',
+    actual: 1,
+    expected: 2,
+    operator: 'strictEqual',
+    diff: 'simple'
+  }
+
+*
+✖ failing test 2 (*ms)
+  AssertionError [ERR_ASSERTION]: This test fails as well
+  
+  3 !== 4
+  
+      *
+      *
+      *
+      *
+      *
+      *
+      * {
+    generatedMessage: false,
+    code: 'ERR_ASSERTION',
+    actual: 3,
+    expected: 4,
+    operator: 'strictEqual',
+    diff: 'simple'
+  }

test/parallel/test-runner-run-bail-isolation-none.mjs

@@ -1,15 +1,44 @@
-import '../common/index.mjs';
+import * as common from '../common/index.mjs';
 import assert from 'node:assert';
-import { run } from 'node:test';
+import { test, run } from 'node:test';
 import * as fixtures from '../common/fixtures.mjs';
 
 const files = [
   fixtures.path('test-runner', 'bail', 'bail-test-1-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-2-fail.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-3-pass.js'),
+  fixtures.path('test-runner', 'bail', 'bail-test-4-pass.js'),
 ];
 
-assert.throws(() => {
-  run({ bail: true, isolation: 'none', files });
-}, {
-  code: 'ERR_INVALID_ARG_VALUE',
-  message: /bail not supported with 'none' isolation/,
+async function runScenario() {
+  const stream = run({ bail: true, isolation: 'process', concurrency: 1, files });
+
+  let testCount = 0;
+  let failCount = 0;
+  let bailCount = 0;
+
+  stream.on('test:bail', common.mustCall(() => {
+    bailCount++;
+  }));
+
+  stream.on('test:pass', () => {
+    testCount++;
+  });
+
+  stream.on('test:fail', () => {
+    failCount++;
+  });
+
+  // eslint-disable-next-line no-unused-vars, no-empty
+  for await (const _ of stream) {}
+
+  return { testCount, failCount, bailCount };
+}
+
+test('it should bail in isolation process mode', async () => {
+  const { testCount, failCount, bailCount } = await runScenario();
+
+  assert.strictEqual(failCount, 2);
+  assert.strictEqual(testCount, 2);
+  assert.strictEqual(bailCount, 1);
 });