From f87b7255cfea3dc5e8881fdebf9af3ea0312888d Mon Sep 17 00:00:00 2001 From: Alejandro Espa Date: Sun, 8 Feb 2026 22:52:20 +0100 Subject: [PATCH 1/3] feat: first commit flaky --- lib/internal/test_runner/harness.js | 3 ++- lib/internal/test_runner/reporter/tap.js | 10 +++++++--- lib/internal/test_runner/reporter/utils.js | 5 ++++- lib/internal/test_runner/test.js | 4 +++- lib/internal/test_runner/utils.js | 4 ++++ package-lock.json | 6 ++++++ 6 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 package-lock.json diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 6b3b13b2c88d65..ecee5cdefaa897 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -61,6 +61,7 @@ function createTestTree(rootTestOptions, globalOptions) { failed: 0, passed: 0, cancelled: 0, + flaky: 0, skipped: 0, todo: 0, topLevel: 0, @@ -377,7 +378,7 @@ function runInParentContext(Factory) { return run(name, options, fn, overrides); }; - ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => { + ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => { test[keyword] = (name, options, fn) => { const overrides = { __proto__: null, diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 01c698871b9134..595c96b48c8970 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -33,12 +33,12 @@ async function * tapReporter(source) { for await (const { type, data } of source) { switch (type) { case 'test:fail': { - yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure); + yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); const location = data.file ? `${data.file}:${data.line}:${data.column}` : null; yield reportDetails(data.nesting, data.details, location); break; } case 'test:pass': - yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure); + yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); yield reportDetails(data.nesting, data.details, null); break; case 'test:plan': @@ -65,7 +65,7 @@ async function * tapReporter(source) { } } -function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) { +function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, flaky) { let line = `${indent(nesting)}${status} ${testNumber}`; if (name) { @@ -78,6 +78,10 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { line += ' # EXPECTED FAILURE'; + //should we use flaky >=0 here? for always printing 0 retries + } else if (flaky !== undefined && flaky > 0) { + const retryText = flaky === 1 ? 're-try' : 're-tries'; + line += ` # FLAKY ${flaky} ${retryText}`; } line += '\n'; diff --git a/lib/internal/test_runner/reporter/utils.js b/lib/internal/test_runner/reporter/utils.js index d90040b9727aa2..4f0f888572c2d1 100644 --- a/lib/internal/test_runner/reporter/utils.js +++ b/lib/internal/test_runner/reporter/utils.js @@ -71,7 +71,7 @@ function formatError(error, indent) { function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') { let color = reporterColorMap[type] ?? colors.white; let symbol = reporterUnicodeSymbolMap[type] ?? ' '; - const { skip, todo, expectFailure } = data; + const { skip, todo, expectFailure, flaky } = data; const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : ''; let title = `${data.name}${duration_ms}`; @@ -87,6 +87,9 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde } } else if (expectFailure !== undefined) { title += ` # EXPECTED FAILURE`; + } else if (flaky !== undefined && flaky > 0) { + const retryText = flaky === 1 ? 're-try' : 're-tries'; + title += ` # FLAKY ${flaky} ${retryText}`; } const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : ''; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index beeb49c1763473..dffa7f0980d876 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -85,6 +85,7 @@ const kTestTimeoutFailure = 'testTimeoutFailure'; const kExpectedFailure = 'expectedFailure'; const kHookFailure = 'hookFailed'; const kDefaultTimeout = null; +const kDefaultFlakyRetries = 20; const noop = FunctionPrototype; const kShouldAbort = Symbol('kShouldAbort'); const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']); @@ -497,7 +498,8 @@ class Test extends AsyncResource { super('Test'); let { fn, name, parent } = options; - const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options; + + const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options; if (typeof fn !== 'function') { fn = noop; diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 5b53342933cdcb..30ded02c25c3de 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -401,6 +401,10 @@ function countCompletedTest(test, harness = test.root.harness) { } else { harness.counters.passed++; } + + if (test.flakyRetries > 0) { + harness.counters.flaky++; + } harness.counters.tests++; } diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000000000..93bc0662b7b834 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "node", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} From 2879fc75d4fb6dbe088066ca510c79f95616c45f Mon Sep 17 00:00:00 2001 From: Alejandro Espa Date: Mon, 23 Mar 2026 23:11:40 +0100 Subject: [PATCH 2/3] feat: flaky tests implemented --- doc/api/test.md | 102 ++++ lib/internal/test_runner/reporter/dot.js | 11 +- lib/internal/test_runner/reporter/junit.js | 131 ++++- lib/internal/test_runner/reporter/tap.js | 80 ++- lib/internal/test_runner/reporter/utils.js | 39 +- lib/internal/test_runner/test.js | 507 ++++++++++++------ lib/internal/test_runner/tests_stream.js | 4 + package-lock.json | 6 - test/fixtures/test-runner/output/flaky.js | 68 +++ .../test-runner/output/flaky.snapshot | 69 +++ test/test-runner/test-output-flaky.mjs | 11 + 11 files changed, 805 insertions(+), 223 deletions(-) delete mode 100644 package-lock.json create mode 100644 test/fixtures/test-runner/output/flaky.js create mode 100644 test/fixtures/test-runner/output/flaky.snapshot create mode 100644 test/test-runner/test-output-flaky.mjs diff --git a/doc/api/test.md b/doc/api/test.md index 40fb08d0d5b181..966d70dd9e3875 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -275,6 +275,56 @@ it.todo('should do the thing', { expectFailure: true }, () => { }); ``` +## Flaky tests + + + +This flag causes a test or suite to be re-run a number of times until it +either passes or has not passed after the final re-try. + +When `flaky` is `true`, the test harness re-tries the test up to the default +number of times (20), inclusive. + +When `flaky` is a positive integer, the test harness re-tries the test up to +the specified number of times, inclusive. + +When `flaky` is falsy (the default), the test harness does not re-try the test. + +When both a suite and an included test specify the `flaky` flag, the +test's `flaky` value wins. + +```js +it.flaky('should do something', () => { + // This test will be retried up to 20 times if it fails +}); + +it('may take several times', { flaky: true }, () => { + // Also retries up to 20 times +}); + +it('may also take several times', { flaky: 5 }, () => { + // Retries up to 5 times +}); + +describe.flaky('flaky suite', () => { + it('inherits flaky from suite', () => { + // Retried up to 20 times (inherited from suite) + }); + + it('not flaky', { flaky: false }, () => { + // Not retried, overrides suite setting + }); +}); +``` + +When a test marked `flaky` passes after retries, the number of re-tries taken +is reported with that test. + +`skip` and `todo` take precedence over `flaky`. + ## `describe()` and `it()` aliases Suites and tests can also be written using the `describe()` and `it()` @@ -1649,6 +1699,16 @@ added: Shorthand for marking a suite as `only`. This is the same as [`suite([name], { only: true }[, fn])`][suite options]. +## `suite.flaky([name][, options][, fn])` + + + +Shorthand for marking a suite as flaky. This is the same as +[`suite([name], { flaky: true }[, fn])`][suite options]. + ## `test([name][, options][, fn])` + +Shorthand for marking a test as flaky, +same as [`test([name], { flaky: true }[, fn])`][it options]. + ## `describe([name][, options][, fn])` Alias for [`suite()`][]. @@ -1782,6 +1857,16 @@ added: Shorthand for marking a suite as `only`. This is the same as [`describe([name], { only: true }[, fn])`][describe options]. +## `describe.flaky([name][, options][, fn])` + + + +Shorthand for marking a suite as flaky. This is the same as +[`describe([name], { flaky: true }[, fn])`][describe options]. + ## `it([name][, options][, fn])` + +Shorthand for marking a test as flaky, +same as [`it([name], { flaky: true }[, fn])`][it options]. + ## `before([fn][, options])` \n`; @@ -44,21 +56,34 @@ function treeToXML(tree) { const attrsString = ArrayPrototypeJoin( ArrayPrototypeMap( ObjectEntries(attrs), - ({ 0: key, 1: value }) => `${key}="${escapeAttribute(String(value))}"`), - ' '); + ({ 0: key, 1: value }) => `${key}="${escapeAttribute(String(value))}"`, + ), + ' ', + ); if (!children?.length) { return `${indent}<${tag} ${attrsString}/>\n`; } - const childrenString = ArrayPrototypeJoin(ArrayPrototypeMap(children ?? [], treeToXML), ''); + const childrenString = ArrayPrototypeJoin( + ArrayPrototypeMap(children ?? [], treeToXML), + '', + ); return `${indent}<${tag} ${attrsString}>\n${childrenString}${indent}\n`; } function isFailure(node) { - return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'failure')) || node?.attrs?.failures; + return ( + (node?.children && + ArrayPrototypeSome(node.children, (c) => c.tag === 'failure')) || + node?.attrs?.failures + ); } function isSkipped(node) { - return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'skipped')) || node?.attrs?.skipped; + return ( + (node?.children && + ArrayPrototypeSome(node.children, (c) => c.tag === 'skipped')) || + node?.attrs?.skipped + ); } module.exports = async function* junitReporter(source) { @@ -93,25 +118,42 @@ module.exports = async function* junitReporter(source) { case 'test:pass': case 'test:fail': { if (!currentSuite) { - startTest({ __proto__: null, data: { __proto__: null, name: 'root', nesting: 0 } }); + startTest({ + __proto__: null, + data: { __proto__: null, name: 'root', nesting: 0 }, + }); } - if (currentSuite.attrs.name !== event.data.name || - currentSuite.nesting !== event.data.nesting) { + if ( + currentSuite.attrs.name !== event.data.name || + currentSuite.nesting !== event.data.nesting + ) { startTest(event); } const currentTest = currentSuite; if (currentSuite?.nesting === event.data.nesting) { currentSuite = currentSuite.parent; } - currentTest.attrs.time = NumberPrototypeToFixed(event.data.details.duration_ms / 1000, 6); - const nonCommentChildren = ArrayPrototypeFilter(currentTest.children, (c) => c.comment == null); + currentTest.attrs.time = NumberPrototypeToFixed( + event.data.details.duration_ms / 1000, + 6, + ); + const nonCommentChildren = ArrayPrototypeFilter( + currentTest.children, + (c) => c.comment == null, + ); if (nonCommentChildren.length > 0) { currentTest.tag = 'testsuite'; currentTest.attrs.disabled = 0; currentTest.attrs.errors = 0; currentTest.attrs.tests = nonCommentChildren.length; - currentTest.attrs.failures = ArrayPrototypeFilter(currentTest.children, isFailure).length; - currentTest.attrs.skipped = ArrayPrototypeFilter(currentTest.children, isSkipped).length; + currentTest.attrs.failures = ArrayPrototypeFilter( + currentTest.children, + isFailure, + ).length; + currentTest.attrs.skipped = ArrayPrototypeFilter( + currentTest.children, + isSkipped, + ).length; currentTest.attrs.hostname = HOSTNAME; } else { currentTest.tag = 'testcase'; @@ -121,14 +163,46 @@ module.exports = async function* junitReporter(source) { } if (event.data.skip) { ArrayPrototypePush(currentTest.children, { - __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', - attrs: { __proto__: null, type: 'skipped', message: event.data.skip }, + __proto__: null, + nesting: event.data.nesting + 1, + tag: 'skipped', + attrs: { + __proto__: null, + type: 'skipped', + message: event.data.skip, + }, }); } if (event.data.todo) { ArrayPrototypePush(currentTest.children, { - __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', - attrs: { __proto__: null, type: 'todo', message: event.data.todo }, + __proto__: null, + nesting: event.data.nesting + 1, + tag: 'skipped', + attrs: { + __proto__: null, + type: 'todo', + message: event.data.todo, + }, + }); + } + if (event.data.flakyRetriedCount > 0) { + ArrayPrototypePush(currentTest.children, { + __proto__: null, + nesting: event.data.nesting + 1, + tag: 'properties', + attrs: { __proto__: null }, + children: [ + { + __proto__: null, + nesting: event.data.nesting + 2, + tag: 'property', + attrs: { + __proto__: null, + name: 'flaky', + value: `${event.data.flakyRetriedCount} retries`, + }, + }, + ], }); } if (event.type === 'test:fail') { @@ -137,7 +211,11 @@ module.exports = async function* junitReporter(source) { __proto__: null, nesting: event.data.nesting + 1, tag: 'failure', - attrs: { __proto__: null, type: error?.failureType || error?.code, message: error?.message.trim() ?? '' }, + attrs: { + __proto__: null, + type: error?.failureType || error?.code, + message: error?.message.trim() ?? '', + }, children: [inspectWithNoCustomRetry(error, inspectOptions)], }); currentTest.failures = 1; @@ -149,10 +227,13 @@ module.exports = async function* junitReporter(source) { case 'test:diagnostic': { const parent = currentSuite?.children ?? roots; ArrayPrototypePush(parent, { - __proto__: null, nesting: event.data.nesting, comment: event.data.message, + __proto__: null, + nesting: event.data.nesting, + comment: event.data.message, }); break; - } default: + } + default: break; } } diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 595c96b48c8970..6578290f714ceb 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -19,7 +19,11 @@ const kDefaultIndent = ' '; // 4 spaces const kFrameStartRegExp = /^ {4}at /; const kLineBreakRegExp = /\n|\r\n/; const kDefaultTAPVersion = 13; -const inspectOptions = { __proto__: null, colors: false, breakLength: Infinity }; +const inspectOptions = { + __proto__: null, + colors: false, + breakLength: Infinity, +}; let testModule; // Lazy loaded due to circular dependency. function lazyLoadTest() { @@ -27,18 +31,38 @@ function lazyLoadTest() { return testModule; } - -async function * tapReporter(source) { +async function* tapReporter(source) { yield `TAP version ${kDefaultTAPVersion}\n`; for await (const { type, data } of source) { switch (type) { case 'test:fail': { - yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); - const location = data.file ? `${data.file}:${data.line}:${data.column}` : null; + yield reportTest( + data.nesting, + data.testNumber, + 'not ok', + data.name, + data.skip, + data.todo, + data.expectFailure, + data.flakyRetriedCount, + ); + const location = data.file + ? `${data.file}:${data.line}:${data.column}` + : null; yield reportDetails(data.nesting, data.details, location); break; - } case 'test:pass': - yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); + } + case 'test:pass': + yield reportTest( + data.nesting, + data.testNumber, + 'ok', + data.name, + data.skip, + data.todo, + data.expectFailure, + data.flakyRetriedCount, + ); yield reportDetails(data.nesting, data.details, null); break; case 'test:plan': @@ -49,23 +73,42 @@ async function * tapReporter(source) { break; case 'test:stderr': case 'test:stdout': { - const lines = RegExpPrototypeSymbolSplit(kLineBreakRegExp, data.message); + const lines = RegExpPrototypeSymbolSplit( + kLineBreakRegExp, + data.message, + ); for (let i = 0; i < lines.length; i++) { if (lines[i].length === 0) continue; yield `# ${tapEscape(lines[i])}\n`; } break; - } case 'test:diagnostic': + } + case 'test:diagnostic': yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`; break; case 'test:coverage': - yield getCoverageReport(indent(data.nesting), data.summary, '# ', '', true); + yield getCoverageReport( + indent(data.nesting), + data.summary, + '# ', + '', + true, + ); break; } } } -function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, flaky) { +function reportTest( + nesting, + testNumber, + status, + name, + skip, + todo, + expectFailure, + flakyRetriedCount, +) { let line = `${indent(nesting)}${status} ${testNumber}`; if (name) { @@ -78,10 +121,9 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { line += ' # EXPECTED FAILURE'; - //should we use flaky >=0 here? for always printing 0 retries - } else if (flaky !== undefined && flaky > 0) { - const retryText = flaky === 1 ? 're-try' : 're-tries'; - line += ` # FLAKY ${flaky} ${retryText}`; + } else if (flakyRetriedCount !== undefined && flakyRetriedCount > 0) { + const retryText = flakyRetriedCount === 1 ? 're-try' : 're-tries'; + line += ` # FLAKY ${flakyRetriedCount} ${retryText}`; } line += '\n'; @@ -117,7 +159,6 @@ function indent(nesting) { return value; } - // In certain places, # and \ need to be escaped as \# and \\. function tapEscape(input) { let result = StringPrototypeReplaceAll(input, '\b', '\\b'); @@ -281,7 +322,12 @@ function jsToYaml(indent, name, value, seen) { } function isAssertionLike(value) { - return value && typeof value === 'object' && 'expected' in value && 'actual' in value; + return ( + value && + typeof value === 'object' && + 'expected' in value && + 'actual' in value + ); } module.exports = tapReporter; diff --git a/lib/internal/test_runner/reporter/utils.js b/lib/internal/test_runner/reporter/utils.js index 4f0f888572c2d1..1986adfb1572a8 100644 --- a/lib/internal/test_runner/reporter/utils.js +++ b/lib/internal/test_runner/reporter/utils.js @@ -17,7 +17,7 @@ const inspectOptions = { }; const reporterUnicodeSymbolMap = { - '__proto__': null, + __proto__: null, 'test:fail': '\u2716 ', 'test:pass': '\u2714 ', 'test:diagnostic': '\u2139 ', @@ -28,7 +28,7 @@ const reporterUnicodeSymbolMap = { }; const reporterColorMap = { - '__proto__': null, + __proto__: null, get 'test:fail'() { return colors.red; }, @@ -38,13 +38,13 @@ const reporterColorMap = { get 'test:diagnostic'() { return colors.blue; }, - get 'info'() { + get info() { return colors.blue; }, - get 'warn'() { + get warn() { return colors.yellow; }, - get 'error'() { + get error() { return colors.red; }, }; @@ -64,15 +64,25 @@ function formatError(error, indent) { RegExpPrototypeSymbolSplit( hardenRegExp(/\r?\n/), inspectWithNoCustomRetry(err, inspectOptions), - ), `\n${indent} `); + ), + `\n${indent} `, + ); return `\n${indent} ${message}\n`; } -function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') { +function formatTestReport( + type, + data, + showErrorDetails = true, + prefix = '', + indent = '', +) { let color = reporterColorMap[type] ?? colors.white; let symbol = reporterUnicodeSymbolMap[type] ?? ' '; - const { skip, todo, expectFailure, flaky } = data; - const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : ''; + const { skip, todo, expectFailure, flakyRetriedCount } = data; + const duration_ms = data.details?.duration_ms + ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` + : ''; let title = `${data.name}${duration_ms}`; if (skip !== undefined) { @@ -87,12 +97,15 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde } } else if (expectFailure !== undefined) { title += ` # EXPECTED FAILURE`; - } else if (flaky !== undefined && flaky > 0) { - const retryText = flaky === 1 ? 're-try' : 're-tries'; - title += ` # FLAKY ${flaky} ${retryText}`; + } else if (flakyRetriedCount !== undefined && flakyRetriedCount > 0) { + const retryText = flakyRetriedCount === 1 ? 're-try' : 're-tries'; + title += ` # FLAKY ${flakyRetriedCount} ${retryText}`; } - const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : ''; + const err = + showErrorDetails && data.details?.error + ? formatError(data.details.error, indent) + : ''; return `${prefix}${indent}${color}${symbol}${title}${colors.white}${err}`; } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index dffa7f0980d876..9b994ba7b714f8 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -12,6 +12,7 @@ const { FunctionPrototype, MathMax, Number, + NumberIsInteger, NumberPrototypeToFixed, ObjectSeal, Promise, @@ -31,17 +32,16 @@ const { SymbolDispose, } = primordials; const { getCallerLocation } = internalBinding('util'); -const { exitCodes: { kGenericUserError } } = internalBinding('errors'); +const { + exitCodes: { kGenericUserError }, +} = internalBinding('errors'); const { addAbortListener } = require('internal/events/abort_listener'); const { queueMicrotask } = require('internal/process/task_queues'); const { AsyncResource } = require('async_hooks'); const { AbortController } = require('internal/abort_controller'); const { AbortError, - codes: { - ERR_INVALID_ARG_TYPE, - ERR_TEST_FAILURE, - }, + codes: { ERR_INVALID_ARG_TYPE, ERR_TEST_FAILURE }, } = require('internal/errors'); const { MockTracker } = require('internal/test_runner/mock/mock'); const { TestsStream } = require('internal/test_runner/tests_stream'); @@ -65,10 +65,7 @@ const { validateOneOf, validateUint32, } = require('internal/validators'); -const { - clearTimeout, - setTimeout, -} = require('timers'); +const { clearTimeout, setTimeout } = require('timers'); const { TIMEOUT_MAX } = require('internal/timers'); const { fileURLToPath } = require('internal/url'); const { relative } = require('path'); @@ -90,8 +87,10 @@ const noop = FunctionPrototype; const kShouldAbort = Symbol('kShouldAbort'); const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']); const kUnwrapErrors = new SafeSet() - .add(kTestCodeFailure).add(kHookFailure) - .add('uncaughtException').add('unhandledRejection'); + .add(kTestCodeFailure) + .add(kHookFailure) + .add('uncaughtException') + .add('unhandledRejection'); let kResistStopPropagation; let assertObj; let findSourceMap; @@ -113,7 +112,9 @@ function lazyAssertObject(harness) { const { SnapshotManager } = require('internal/test_runner/snapshot'); assertObj = getAssertionMap(); - harness.snapshotManager = new SnapshotManager(harness.config.updateSnapshots); + harness.snapshotManager = new SnapshotManager( + harness.config.updateSnapshots, + ); if (!assertObj.has('snapshot')) { assertObj.set('snapshot', harness.snapshotManager.createAssert()); @@ -137,12 +138,16 @@ function stopTest(timeout, signal) { } else { timer = setTimeout(deferred.resolve, timeout); timer.unref(); - setOwnProperty(deferred, 'promise', PromisePrototypeThen(deferred.promise, () => { - throw new ERR_TEST_FAILURE( - `test timed out after ${timeout}ms`, - kTestTimeoutFailure, - ); - })); + setOwnProperty( + deferred, + 'promise', + PromisePrototypeThen(deferred.promise, () => { + throw new ERR_TEST_FAILURE( + `test timed out after ${timeout}ms`, + kTestTimeoutFailure, + ); + }), + ); disposeFunction = () => { abortListener[SymbolDispose](); @@ -155,15 +160,21 @@ function stopTest(timeout, signal) { } function testMatchesPattern(test, patterns) { - const matchesByNameOrParent = ArrayPrototypeSome(patterns, (re) => - RegExpPrototypeExec(re, test.name) !== null, - ) || (test.parent && testMatchesPattern(test.parent, patterns)); + const matchesByNameOrParent = + ArrayPrototypeSome( + patterns, + (re) => RegExpPrototypeExec(re, test.name) !== null, + ) || + (test.parent && testMatchesPattern(test.parent, patterns)); if (matchesByNameOrParent) return true; - const testNameWithAncestors = StringPrototypeTrim(test.getTestNameWithAncestors()); + const testNameWithAncestors = StringPrototypeTrim( + test.getTestNameWithAncestors(), + ); - return ArrayPrototypeSome(patterns, (re) => - RegExpPrototypeExec(re, testNameWithAncestors) !== null, + return ArrayPrototypeSome( + patterns, + (re) => RegExpPrototypeExec(re, testNameWithAncestors) !== null, ); } @@ -186,7 +197,11 @@ class TestPlan { validateNumber(wait, 'options.wait', 0, TIMEOUT_MAX); this.wait = wait; } else if (wait !== undefined) { - throw new ERR_INVALID_ARG_TYPE('options.wait', ['boolean', 'number'], wait); + throw new ERR_INVALID_ARG_TYPE( + 'options.wait', + ['boolean', 'number'], + wait, + ); } } @@ -249,7 +264,6 @@ class TestPlan { } } - class TestContext { #assert; #test; @@ -363,7 +377,11 @@ class TestContext { const subtest = this.#test.createSubtest( // eslint-disable-next-line no-use-before-define - Test, name, options, fn, overrides, + Test, + name, + options, + fn, + overrides, ); return subtest.start(); @@ -413,10 +431,7 @@ class TestContext { validateFunction(condition, 'condition'); validateObject(options, 'options'); - const { - interval = 50, - timeout = 1000, - } = options; + const { interval = 50, timeout = 1000 } = options; validateNumber(interval, 'options.interval', 0, TIMEOUT_MAX); validateNumber(timeout, 'options.timeout', 0, TIMEOUT_MAX); @@ -499,7 +514,19 @@ class Test extends AsyncResource { let { fn, name, parent } = options; - const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options; + const { + concurrency, + entryFile, + expectFailure, + flaky, + loc, + only, + timeout, + todo, + skip, + signal, + plan, + } = options; if (typeof fn !== 'function') { fn = noop; @@ -536,9 +563,10 @@ class Test extends AsyncResource { this.entryFile = entryFile; this.testDisambiguator = new SafeMap(); } else { - const nesting = parent.parent === null ? parent.nesting : - parent.nesting + 1; - const { config, isFilteringByName, isFilteringByOnly } = parent.root.harness; + const nesting = + parent.parent === null ? parent.nesting : parent.nesting + 1; + const { config, isFilteringByName, isFilteringByOnly } = + parent.root.harness; this.root = parent.root; this.harness = null; @@ -555,7 +583,11 @@ class Test extends AsyncResource { if (isFilteringByName) { this.filteredByName = this.willBeFilteredByName(); if (!this.filteredByName) { - for (let t = this.parent; t !== null && t.filteredByName; t = t.parent) { + for ( + let t = this.parent; + t !== null && t.filteredByName; + t = t.parent + ) { t.filteredByName = false; } } @@ -569,7 +601,11 @@ class Test extends AsyncResource { this.parent.runOnlySubtests = true; if (this.parent === this.root || this.parent.startTime === null) { - for (let t = this.parent; t !== null && !t.hasOnlyTests; t = t.parent) { + for ( + let t = this.parent; + t !== null && !t.hasOnlyTests; + t = t.parent + ) { t.hasOnlyTests = true; } } @@ -591,8 +627,8 @@ class Test extends AsyncResource { case 'boolean': if (concurrency) { - this.concurrency = parent === null ? - MathMax(availableParallelism() - 1, 1) : Infinity; + this.concurrency = + parent === null ? MathMax(availableParallelism() - 1, 1) : Infinity; } else { this.concurrency = 1; } @@ -600,7 +636,11 @@ class Test extends AsyncResource { default: if (concurrency != null) - throw new ERR_INVALID_ARG_TYPE('options.concurrency', ['boolean', 'number'], concurrency); + throw new ERR_INVALID_ARG_TYPE( + 'options.concurrency', + ['boolean', 'number'], + concurrency, + ); } if (timeout != null && timeout !== Infinity) { @@ -624,14 +664,14 @@ class Test extends AsyncResource { validateAbortSignal(signal, 'options.signal'); if (signal) { - kResistStopPropagation ??= require('internal/event_target').kResistStopPropagation; + kResistStopPropagation ??= + require('internal/event_target').kResistStopPropagation; } - this.outerSignal?.addEventListener( - 'abort', - this.#abortHandler, - { __proto__: null, [kResistStopPropagation]: true }, - ); + this.outerSignal?.addEventListener('abort', this.#abortHandler, { + __proto__: null, + [kResistStopPropagation]: true, + }); this.fn = fn; this.mock = null; @@ -639,6 +679,24 @@ class Test extends AsyncResource { this.expectedAssertions = plan; this.cancelled = false; this.expectFailure = expectFailure !== undefined && expectFailure !== false; + + // Validate and process flaky option + if (flaky != null && flaky !== false) { + this.flakyRetries = flaky === true ? kDefaultFlakyRetries : flaky; + if (!NumberIsInteger(this.flakyRetries) || this.flakyRetries < 1) { + throw new ERR_INVALID_ARG_TYPE( + 'options.flaky', + ['boolean', 'positive integer'], + flaky, + ); + } + } else if (flaky === undefined && this.parent?.flakyRetries > 0) { + // Inherit flaky from parent suite + this.flakyRetries = this.parent.flakyRetries; + } else { + this.flakyRetries = 0; + } + this.retriesTaken = 0; this.skipped = skip !== undefined && skip !== false; this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo; this.startTime = null; @@ -647,8 +705,8 @@ class Test extends AsyncResource { this.error = null; this.attempt = undefined; this.passedAttempt = undefined; - this.message = typeof skip === 'string' ? skip : - typeof todo === 'string' ? todo : null; + this.message = + typeof skip === 'string' ? skip : typeof todo === 'string' ? todo : null; this.activeSubtests = 0; this.pendingSubtests = []; this.readySubtests = new SafeMap(); @@ -702,16 +760,24 @@ class Test extends AsyncResource { this.root.testDisambiguator.set(testIdentifier, 1); } this.attempt = this.root.harness.previousRuns.length; - const previousAttempt = this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]; + const previousAttempt = + this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]; if (previousAttempt != null) { this.passedAttempt = previousAttempt.passed_on_attempt; this.fn = () => { for (let i = 0; i < (previousAttempt.children?.length ?? 0); i++) { const child = previousAttempt.children[i]; - this.createSubtest(Test, child.name, { __proto__: null }, noop, { - __proto__: null, - loc: [child.line, child.column, child.file], - }, noop).start(); + this.createSubtest( + Test, + child.name, + { __proto__: null }, + noop, + { + __proto__: null, + loc: [child.line, child.column, child.file], + }, + noop, + ).start(); } }; } @@ -729,10 +795,16 @@ class Test extends AsyncResource { return; } - if (this.root.harness.isFilteringByOnly && !this.only && !this.hasOnlyTests) { - if (this.parent.runOnlySubtests || - this.parent.hasOnlyTests || - this.only === false) { + if ( + this.root.harness.isFilteringByOnly && + !this.only && + !this.hasOnlyTests + ) { + if ( + this.parent.runOnlySubtests || + this.parent.hasOnlyTests || + this.only === false + ) { this.filtered = true; } } @@ -785,7 +857,12 @@ class Test extends AsyncResource { while (this.pendingSubtests.length > 0 && this.hasConcurrency()) { const deferred = ArrayPrototypeShift(this.pendingSubtests); const test = deferred.test; - test.reporter.dequeue(test.nesting, test.loc, test.name, this.reportedType); + test.reporter.dequeue( + test.nesting, + test.loc, + test.name, + this.reportedType, + ); await test.run(); deferred.resolve(); } @@ -798,8 +875,10 @@ class Test extends AsyncResource { addReadySubtest(subtest) { this.readySubtests.set(subtest.childNumber, subtest); - if (this.unfinishedSubtests.delete(subtest) && - this.unfinishedSubtests.size === 0) { + if ( + this.unfinishedSubtests.delete(subtest) && + this.unfinishedSubtests.size === 0 + ) { this.subtestsPromise.resolve(); } } @@ -863,7 +942,14 @@ class Test extends AsyncResource { } } - const test = new Factory({ __proto__: null, fn, name, parent, ...options, ...overrides }); + const test = new Factory({ + __proto__: null, + fn, + name, + parent, + ...options, + ...overrides, + }); if (parent.waitingOn === 0) { parent.waitingOn = test.childNumber; @@ -884,7 +970,8 @@ class Test extends AsyncResource { } #abortHandler = () => { - const error = this.outerSignal?.reason || new AbortError('The test was aborted'); + const error = + this.outerSignal?.reason || new AbortError('The test was aborted'); error.failureType = kAborted; this.#cancel(error); }; @@ -894,11 +981,12 @@ class Test extends AsyncResource { return; } - this.fail(error || - new ERR_TEST_FAILURE( - 'test did not finish before its parent and was cancelled', - kCancelledByParent, - ), + this.fail( + error || + new ERR_TEST_FAILURE( + 'test did not finish before its parent and was cancelled', + kCancelledByParent, + ), ); this.cancelled = true; this.abortController.abort(); @@ -914,7 +1002,8 @@ class Test extends AsyncResource { if (this.parent.hooks.afterEach.length > 0) { ArrayPrototypePushApply( - this.hooks.afterEach, ArrayPrototypeSlice(this.parent.hooks.afterEach), + this.hooks.afterEach, + ArrayPrototypeSlice(this.parent.hooks.afterEach), ); } } @@ -938,7 +1027,12 @@ class Test extends AsyncResource { // afterEach hooks for the current test should run in the order that they // are created. However, the current test's afterEach hooks should run // prior to any ancestor afterEach hooks. - ArrayPrototypeSplice(this.hooks[name], this.hooks.ownAfterEachCount, 0, hook); + ArrayPrototypeSplice( + this.hooks[name], + this.hooks.ownAfterEachCount, + 0, + hook, + ); this.hooks.ownAfterEachCount++; } else { ArrayPrototypePush(this.hooks[name], hook); @@ -1041,7 +1135,10 @@ class Test extends AsyncResource { } } } catch (err) { - const error = new ERR_TEST_FAILURE(`failed running ${hook} hook`, kHookFailure); + const error = new ERR_TEST_FAILURE( + `failed running ${hook} hook`, + kHookFailure, + ); error.cause = isTestFailureError(err) ? err.cause : err; throw error; } @@ -1095,53 +1192,69 @@ class Test extends AsyncResource { await this.parent.runHook('beforeEach', hookArgs); } stopPromise = stopTest(this.timeout, this.signal); - const runArgs = ArrayPrototypeSlice(args); - ArrayPrototypeUnshift(runArgs, this.fn, ctx); - const promises = []; - if (this.fn.length === runArgs.length - 1) { - // This test is using legacy Node.js error-first callbacks. - const { promise, cb } = createDeferredCallback(); - ArrayPrototypePush(runArgs, cb); - - const ret = ReflectApply(this.runInAsyncScope, this, runArgs); - - if (isPromise(ret)) { - this.fail(new ERR_TEST_FAILURE( - 'passed a callback but also returned a Promise', - kCallbackAndPromisePresent, - )); - ArrayPrototypePush(promises, ret); - } else { - ArrayPrototypePush(promises, PromiseResolve(promise)); + const maxAttempts = this.flakyRetries > 0 ? this.flakyRetries + 1 : 1; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + if (attempt > 1) { + this.error = null; + this.passed = false; } - } else { - // This test is synchronous or using Promises. - const promise = ReflectApply(this.runInAsyncScope, this, runArgs); - ArrayPrototypePush(promises, PromiseResolve(promise)); - } - ArrayPrototypePush(promises, stopPromise); + try { + const runArgs = ArrayPrototypeSlice(args); + ArrayPrototypeUnshift(runArgs, this.fn, ctx); + + const promises = []; + if (this.fn.length === runArgs.length - 1) { + const { promise, cb } = createDeferredCallback(); + ArrayPrototypePush(runArgs, cb); + + const ret = ReflectApply(this.runInAsyncScope, this, runArgs); + + if (isPromise(ret)) { + this.fail( + new ERR_TEST_FAILURE( + 'passed a callback but also returned a Promise', + kCallbackAndPromisePresent, + ), + ); + ArrayPrototypePush(promises, ret); + } else { + ArrayPrototypePush(promises, PromiseResolve(promise)); + } + } else { + const promise = ReflectApply(this.runInAsyncScope, this, runArgs); + ArrayPrototypePush(promises, PromiseResolve(promise)); + } - // Wait for the race to finish - await SafePromiseRace(promises); + ArrayPrototypePush(promises, stopPromise); - this[kShouldAbort](); + await SafePromiseRace(promises); - if (this.subtestsPromise !== null) { - await SafePromiseRace([this.subtestsPromise.promise, stopPromise]); - } + this[kShouldAbort](); + + if (this.subtestsPromise !== null) { + await SafePromiseRace([this.subtestsPromise.promise, stopPromise]); + } + + if (this.plan !== null) { + const planPromise = this.plan?.check(); + if (planPromise) { + await SafePromiseRace([planPromise, stopPromise]); + } + } - if (this.plan !== null) { - const planPromise = this.plan?.check(); - // If the plan returns a promise, it means that it is waiting for more assertions to be made before - // continuing. - if (planPromise) { - await SafePromiseRace([planPromise, stopPromise]); + this.pass(); + break; + } catch (attemptErr) { + if (attempt < maxAttempts) { + this.retriesTaken = attempt; + continue; + } + throw attemptErr; } } - - this.pass(); await afterEach(); await after(); } catch (err) { @@ -1154,8 +1267,16 @@ class Test extends AsyncResource { } else { this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure)); } - try { await afterEach(); } catch { /* test is already failing, let's ignore the error */ } - try { await after(); } catch { /* Ignore error. */ } + try { + await afterEach(); + } catch { + /* test is already failing, let's ignore the error */ + } + try { + await after(); + } catch { + /* Ignore error. */ + } } finally { stopPromise?.[SymbolDispose](); @@ -1186,15 +1307,21 @@ class Test extends AsyncResource { for (let i = 0; i < reporterScope.reporters.length; i++) { const { destination } = reporterScope.reporters[i]; - ArrayPrototypePush(promises, new Promise((resolve) => { - destination.on('unpipe', () => { - if (!destination.closed && typeof destination.close === 'function') { - destination.close(resolve); - } else { - resolve(); - } - }); - })); + ArrayPrototypePush( + promises, + new Promise((resolve) => { + destination.on('unpipe', () => { + if ( + !destination.closed && + typeof destination.close === 'function' + ) { + destination.close(resolve); + } else { + resolve(); + } + }); + }), + ); } this.harness.teardown(); @@ -1240,7 +1367,14 @@ class Test extends AsyncResource { const report = this.getReportDetails(); report.details.passed = this.passed; this.testNumber ||= ++this.parent.outputSubtestCount; - this.reporter.complete(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + this.reporter.complete( + this.nesting, + this.loc, + this.testNumber, + this.name, + report.details, + report.directive, + ); this.parent.activeSubtests--; } @@ -1248,13 +1382,7 @@ class Test extends AsyncResource { this.parent.processReadySubtestRange(false); this.parent.processPendingSubtests(); } else if (!this.reported) { - const { - diagnostics, - harness, - loc, - nesting, - reporter, - } = this; + const { diagnostics, harness, loc, nesting, reporter } = this; this.reported = true; reporter.plan(nesting, loc, harness.counters.topLevel); @@ -1271,21 +1399,38 @@ class Test extends AsyncResource { reporter.diagnostic(nesting, loc, `suites ${harness.counters.suites}`); reporter.diagnostic(nesting, loc, `pass ${harness.counters.passed}`); reporter.diagnostic(nesting, loc, `fail ${harness.counters.failed}`); - reporter.diagnostic(nesting, loc, `cancelled ${harness.counters.cancelled}`); + reporter.diagnostic( + nesting, + loc, + `cancelled ${harness.counters.cancelled}`, + ); reporter.diagnostic(nesting, loc, `skipped ${harness.counters.skipped}`); reporter.diagnostic(nesting, loc, `todo ${harness.counters.todo}`); + reporter.diagnostic(nesting, loc, `flaky ${harness.counters.flaky}`); reporter.diagnostic(nesting, loc, `duration_ms ${duration}`); if (coverage) { const coverages = [ - { __proto__: null, actual: coverage.totals.coveredLinePercent, - threshold: this.config.lineCoverage, name: 'line' }, - - { __proto__: null, actual: coverage.totals.coveredBranchPercent, - threshold: this.config.branchCoverage, name: 'branch' }, - - { __proto__: null, actual: coverage.totals.coveredFunctionPercent, - threshold: this.config.functionCoverage, name: 'function' }, + { + __proto__: null, + actual: coverage.totals.coveredLinePercent, + threshold: this.config.lineCoverage, + name: 'line', + }, + + { + __proto__: null, + actual: coverage.totals.coveredBranchPercent, + threshold: this.config.branchCoverage, + name: 'branch', + }, + + { + __proto__: null, + actual: coverage.totals.coveredFunctionPercent, + threshold: this.config.functionCoverage, + name: 'function', + }, ]; for (let i = 0; i < coverages.length; i++) { @@ -1293,7 +1438,12 @@ class Test extends AsyncResource { if (actual < threshold) { harness.success = false; process.exitCode = kGenericUserError; - reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`, 'error'); + reporter.diagnostic( + nesting, + loc, + `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`, + 'error', + ); } } @@ -1301,7 +1451,11 @@ class Test extends AsyncResource { } reporter.summary( - nesting, loc?.file, harness.success, harness.counters, duration, + nesting, + loc?.file, + harness.success, + harness.counters, + duration, ); if (harness.watching) { @@ -1315,10 +1469,11 @@ class Test extends AsyncResource { } isClearToSend() { - return this.parent === null || - ( - this.parent.waitingOn === this.childNumber && this.parent.isClearToSend() - ); + return ( + this.parent === null || + (this.parent.waitingOn === this.childNumber && + this.parent.isClearToSend()) + ); } finalize() { @@ -1337,8 +1492,10 @@ class Test extends AsyncResource { this.parent.waitingOn++; this.finished = true; - if (this.parent === this.root && - this.root.waitingOn > this.root.subtests.length) { + if ( + this.parent === this.root && + this.root.waitingOn > this.root.subtests.length + ) { // At this point all of the tests have finished running. However, there // might be ref'ed handles keeping the event loop alive. This gives the // global after() hook a chance to clean them up. The user may also @@ -1362,6 +1519,8 @@ class Test extends AsyncResource { directive = this.reporter.getTodo(this.message); } else if (this.expectFailure) { directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure + } else if (this.flakyRetries > 0 && this.retriesTaken > 0 && this.passed) { + directive = this.reporter.getFlaky(this.retriesTaken); } if (this.reportedType) { @@ -1383,16 +1542,34 @@ class Test extends AsyncResource { report() { countCompletedTest(this); if (this.outputSubtestCount > 0) { - this.reporter.plan(this.subtests[0].nesting, this.loc, this.outputSubtestCount); + this.reporter.plan( + this.subtests[0].nesting, + this.loc, + this.outputSubtestCount, + ); } else { this.reportStarted(); } const report = this.getReportDetails(); if (this.passed) { - this.reporter.ok(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + this.reporter.ok( + this.nesting, + this.loc, + this.testNumber, + this.name, + report.details, + report.directive, + ); } else { - this.reporter.fail(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + this.reporter.fail( + this.nesting, + this.loc, + this.testNumber, + this.name, + report.details, + report.directive, + ); } for (let i = 0; i < this.diagnostics.length; i++) { @@ -1459,11 +1636,18 @@ class TestHook extends Test { } this.endTime ??= hrtime(); - parent.reporter.fail(0, loc, parent.subtests.length + 1, loc.file, { - __proto__: null, - duration_ms: this.duration(), - error, - }, undefined); + parent.reporter.fail( + 0, + loc, + parent.subtests.length + 1, + loc.file, + { + __proto__: null, + duration_ms: this.duration(), + error, + }, + undefined, + ); } } } @@ -1476,9 +1660,11 @@ class Suite extends Test { this.timeout = null; } - if (this.config.testNamePatterns !== null && - this.config.testSkipPatterns !== null && - !options.skip) { + if ( + this.config.testNamePatterns !== null && + this.config.testSkipPatterns !== null && + !options.skip + ) { this.fn = options.fn || this.fn; this.skipped = false; } @@ -1510,7 +1696,10 @@ class Suite extends Test { const hookArgs = this.getRunArgs(); let stopPromise; - const after = runOnce(() => this.runHook('after', hookArgs), kRunOnceOptions); + const after = runOnce( + () => this.runHook('after', hookArgs), + kRunOnceOptions, + ); try { this.parent.activeSubtests++; await this.buildSuite; @@ -1537,7 +1726,11 @@ class Suite extends Test { this.pass(); } catch (err) { - try { await after(); } catch { /* suite is already failing */ } + try { + await after(); + } catch { + /* suite is already failing */ + } if (isTestFailureError(err)) { this.fail(err); } else { diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js index 7b64487696f53f..5e5ed10044165e 100644 --- a/lib/internal/test_runner/tests_stream.js +++ b/lib/internal/test_runner/tests_stream.js @@ -91,6 +91,10 @@ class TestsStream extends Readable { return { __proto__: null, expectFailure: expectation ?? true }; } + getFlaky(retriesTaken = undefined) { + return { __proto__: null, flakyRetriedCount: retriesTaken ?? 0 }; + } + enqueue(nesting, loc, name, type) { this[kEmitMessage]('test:enqueue', { __proto__: null, diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 93bc0662b7b834..00000000000000 --- a/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "node", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/test/fixtures/test-runner/output/flaky.js b/test/fixtures/test-runner/output/flaky.js new file mode 100644 index 00000000000000..69a8973159d5e0 --- /dev/null +++ b/test/fixtures/test-runner/output/flaky.js @@ -0,0 +1,68 @@ +// Flags: --test-reporter=tap +'use strict'; +require('../../../common'); +const { test, describe, it } = require('node:test'); + +// Track invocation count for flaky tests +let flakyTestInvocations = 0; +let flakyTestMethodInvocations = 0; +let flakyTestAlwaysFails = 0; + +// Test that passes after 2 retries (3rd attempt succeeds) +test('flaky test that passes on retry', { flaky: 5 }, () => { + flakyTestInvocations++; + if (flakyTestInvocations < 3) { + throw new Error('Simulated failure'); + } + // Passes on 3rd attempt +}); + +// Test using method syntax +it.flaky('flaky test using method syntax', () => { + flakyTestMethodInvocations++; + if (flakyTestMethodInvocations < 2) { + throw new Error('Simulated failure'); + } + // Passes on 2nd attempt +}); + +// Test that always fails (exhausts retries) +test('flaky test that always fails', { flaky: 3 }, () => { + flakyTestAlwaysFails++; + throw new Error(`Always fails (attempt ${flakyTestAlwaysFails})`); +}); + +// Test with flaky: true (should use default 20) +test('flaky test with true value', { flaky: true }, () => { + // This one passes immediately +}); + +// Suite with flaky option +describe.flaky('flaky suite', () => { + it('inherits flaky from suite', () => { + // Passes immediately + }); + + it('override flaky in test', { flaky: false }, () => { + // Not flaky despite suite being flaky + }); +}); + +// Invalid flaky values should error +try { + test('invalid flaky value -1', { flaky: -1 }, () => {}); +} catch (err) { + console.log('Expected error for negative flaky:', err.message); +} + +try { + test('invalid flaky value 0', { flaky: 0 }, () => {}); +} catch (err) { + console.log('Expected error for zero flaky:', err.message); +} + +try { + test('invalid flaky value string', { flaky: 'invalid' }, () => {}); +} catch (err) { + console.log('Expected error for string flaky:', err.message); +} diff --git a/test/fixtures/test-runner/output/flaky.snapshot b/test/fixtures/test-runner/output/flaky.snapshot new file mode 100644 index 00000000000000..5d2b8aeaf14849 --- /dev/null +++ b/test/fixtures/test-runner/output/flaky.snapshot @@ -0,0 +1,69 @@ +TAP version 13 +# Subtest: flaky test that passes on retry +ok 1 - flaky test that passes on retry # FLAKY 2 re-tries + --- + duration_ms: * + type: 'test' + ... +# Subtest: flaky test using method syntax +ok 2 - flaky test using method syntax # FLAKY 1 re-try + --- + duration_ms: * + type: 'test' + ... +# Subtest: flaky test that always fails +not ok 3 - flaky test that always fails + --- + duration_ms: * + type: 'test' + location: '/test/fixtures/test-runner/output/flaky.js:(LINE):1' + failureType: 'testCodeFailure' + error: 'Always fails (attempt 4)' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: flaky test with true value +ok 4 - flaky test with true value + --- + duration_ms: * + type: 'test' + ... +# Subtest: flaky suite + # Subtest: inherits flaky from suite + ok 1 - inherits flaky from suite + --- + duration_ms: * + type: 'test' + ... + # Subtest: override flaky in test + ok 2 - override flaky in test + --- + duration_ms: * + type: 'test' + ... + 1..2 +ok 5 - flaky suite + --- + duration_ms: * + type: 'suite' + ... +Expected error for negative flaky: * +Expected error for zero flaky: * +Expected error for string flaky: * +1..5 +# tests 6 +# suites 1 +# pass 5 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# flaky 5 +# duration_ms * diff --git a/test/test-runner/test-output-flaky.mjs b/test/test-runner/test-output-flaky.mjs new file mode 100644 index 00000000000000..55ac25c8dde232 --- /dev/null +++ b/test/test-runner/test-output-flaky.mjs @@ -0,0 +1,11 @@ +// Test that the output of test-runner/output/flaky.js matches test-runner/output/flaky.snapshot +import '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { spawnAndAssert, defaultTransform, ensureCwdIsProjectRoot } from '../common/assertSnapshot.js'; + +ensureCwdIsProjectRoot(); +await spawnAndAssert( + fixtures.path('test-runner/output/flaky.js'), + defaultTransform, + { flags: ['--test-reporter=tap'] }, +); From 0df94baeb489de7cc8c7bab8ef17f7b18a72ecd5 Mon Sep 17 00:00:00 2001 From: Alejandro Espa Date: Mon, 13 Apr 2026 12:18:46 +0200 Subject: [PATCH 3/3] feat: pr comments fixed --- doc/api/test.md | 9 +- lib/internal/test_runner/reporter/dot.js | 7 +- lib/internal/test_runner/reporter/junit.js | 115 ++---- lib/internal/test_runner/reporter/tap.js | 79 +--- lib/internal/test_runner/reporter/utils.js | 39 +- lib/internal/test_runner/test.js | 409 +++++++-------------- lib/internal/test_runner/tests_stream.js | 2 +- 7 files changed, 192 insertions(+), 468 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index 966d70dd9e3875..1fd109b087b812 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -277,11 +277,6 @@ it.todo('should do the thing', { expectFailure: true }, () => { ## Flaky tests - - This flag causes a test or suite to be re-run a number of times until it either passes or has not passed after the final re-try. @@ -3437,7 +3432,7 @@ Emitted when a test is enqueued for execution. * `testNumber` {number} The ordinal number of the test. * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called - * `flakyRetriedCount` {number|undefined} The number of retries taken for a + * `retryCount` {number|undefined} The number of retries taken for a flaky test. Present when a test is marked as flaky. Emitted when a test fails. @@ -3467,7 +3462,7 @@ The corresponding execution ordered event is `'test:complete'`. * `testNumber` {number} The ordinal number of the test. * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called - * `flakyRetriedCount` {number|undefined} The number of retries taken for a + * `retryCount` {number|undefined} The number of retries taken for a flaky test. Present when a test is marked as flaky and passed after retries. Emitted when a test passes. diff --git a/lib/internal/test_runner/reporter/dot.js b/lib/internal/test_runner/reporter/dot.js index c3208620d4e9fe..ceaa749babcaf9 100644 --- a/lib/internal/test_runner/reporter/dot.js +++ b/lib/internal/test_runner/reporter/dot.js @@ -1,5 +1,8 @@ 'use strict'; -const { ArrayPrototypePush, MathMax } = primordials; +const { + ArrayPrototypePush, + MathMax, +} = primordials; const colors = require('internal/util/colors'); const { formatTestReport } = require('internal/test_runner/reporter/utils'); @@ -9,7 +12,7 @@ module.exports = async function* dot(source) { const failedTests = []; for await (const { type, data } of source) { if (type === 'test:pass') { - if (data.flakyRetriedCount > 0) { + if (data.retryCount > 0) { yield `${colors.yellow}F${colors.reset}`; } else { yield `${colors.green}.${colors.reset}`; diff --git a/lib/internal/test_runner/reporter/junit.js b/lib/internal/test_runner/reporter/junit.js index 33f1dc1c71c0f6..cdefc9e88078e1 100644 --- a/lib/internal/test_runner/reporter/junit.js +++ b/lib/internal/test_runner/reporter/junit.js @@ -15,29 +15,15 @@ const { const { inspectWithNoCustomRetry } = require('internal/errors'); const { hostname } = require('os'); -const inspectOptions = { - __proto__: null, - colors: false, - breakLength: Infinity, -}; +const inspectOptions = { __proto__: null, colors: false, breakLength: Infinity }; const HOSTNAME = hostname(); function escapeAttribute(s = '') { - return escapeContent( - RegExpPrototypeSymbolReplace( - /"/g, - RegExpPrototypeSymbolReplace(/\n/g, s, ' '), - '"', - ), - ); + return escapeContent(RegExpPrototypeSymbolReplace(/"/g, RegExpPrototypeSymbolReplace(/\n/g, s, ' '), '"')); } function escapeContent(s = '') { - return RegExpPrototypeSymbolReplace( - /\n`; @@ -56,34 +44,21 @@ function treeToXML(tree) { const attrsString = ArrayPrototypeJoin( ArrayPrototypeMap( ObjectEntries(attrs), - ({ 0: key, 1: value }) => `${key}="${escapeAttribute(String(value))}"`, - ), - ' ', - ); + ({ 0: key, 1: value }) => `${key}="${escapeAttribute(String(value))}"`), + ' '); if (!children?.length) { return `${indent}<${tag} ${attrsString}/>\n`; } - const childrenString = ArrayPrototypeJoin( - ArrayPrototypeMap(children ?? [], treeToXML), - '', - ); + const childrenString = ArrayPrototypeJoin(ArrayPrototypeMap(children ?? [], treeToXML), ''); return `${indent}<${tag} ${attrsString}>\n${childrenString}${indent}\n`; } function isFailure(node) { - return ( - (node?.children && - ArrayPrototypeSome(node.children, (c) => c.tag === 'failure')) || - node?.attrs?.failures - ); + return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'failure')) || node?.attrs?.failures; } function isSkipped(node) { - return ( - (node?.children && - ArrayPrototypeSome(node.children, (c) => c.tag === 'skipped')) || - node?.attrs?.skipped - ); + return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'skipped')) || node?.attrs?.skipped; } module.exports = async function* junitReporter(source) { @@ -118,42 +93,25 @@ module.exports = async function* junitReporter(source) { case 'test:pass': case 'test:fail': { if (!currentSuite) { - startTest({ - __proto__: null, - data: { __proto__: null, name: 'root', nesting: 0 }, - }); + startTest({ __proto__: null, data: { __proto__: null, name: 'root', nesting: 0 } }); } - if ( - currentSuite.attrs.name !== event.data.name || - currentSuite.nesting !== event.data.nesting - ) { + if (currentSuite.attrs.name !== event.data.name || + currentSuite.nesting !== event.data.nesting) { startTest(event); } const currentTest = currentSuite; if (currentSuite?.nesting === event.data.nesting) { currentSuite = currentSuite.parent; } - currentTest.attrs.time = NumberPrototypeToFixed( - event.data.details.duration_ms / 1000, - 6, - ); - const nonCommentChildren = ArrayPrototypeFilter( - currentTest.children, - (c) => c.comment == null, - ); + currentTest.attrs.time = NumberPrototypeToFixed(event.data.details.duration_ms / 1000, 6); + const nonCommentChildren = ArrayPrototypeFilter(currentTest.children, (c) => c.comment == null); if (nonCommentChildren.length > 0) { currentTest.tag = 'testsuite'; currentTest.attrs.disabled = 0; currentTest.attrs.errors = 0; currentTest.attrs.tests = nonCommentChildren.length; - currentTest.attrs.failures = ArrayPrototypeFilter( - currentTest.children, - isFailure, - ).length; - currentTest.attrs.skipped = ArrayPrototypeFilter( - currentTest.children, - isSkipped, - ).length; + currentTest.attrs.failures = ArrayPrototypeFilter(currentTest.children, isFailure).length; + currentTest.attrs.skipped = ArrayPrototypeFilter(currentTest.children, isSkipped).length; currentTest.attrs.hostname = HOSTNAME; } else { currentTest.tag = 'testcase'; @@ -163,29 +121,17 @@ module.exports = async function* junitReporter(source) { } if (event.data.skip) { ArrayPrototypePush(currentTest.children, { - __proto__: null, - nesting: event.data.nesting + 1, - tag: 'skipped', - attrs: { - __proto__: null, - type: 'skipped', - message: event.data.skip, - }, + __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', + attrs: { __proto__: null, type: 'skipped', message: event.data.skip }, }); } if (event.data.todo) { ArrayPrototypePush(currentTest.children, { - __proto__: null, - nesting: event.data.nesting + 1, - tag: 'skipped', - attrs: { - __proto__: null, - type: 'todo', - message: event.data.todo, - }, + __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', + attrs: { __proto__: null, type: 'todo', message: event.data.todo }, }); } - if (event.data.flakyRetriedCount > 0) { + if (event.data.retryCount > 0) { ArrayPrototypePush(currentTest.children, { __proto__: null, nesting: event.data.nesting + 1, @@ -199,7 +145,7 @@ module.exports = async function* junitReporter(source) { attrs: { __proto__: null, name: 'flaky', - value: `${event.data.flakyRetriedCount} retries`, + value: `${event.data.retryCount} retries`, }, }, ], @@ -211,11 +157,7 @@ module.exports = async function* junitReporter(source) { __proto__: null, nesting: event.data.nesting + 1, tag: 'failure', - attrs: { - __proto__: null, - type: error?.failureType || error?.code, - message: error?.message.trim() ?? '', - }, + attrs: { __proto__: null, type: error?.failureType || error?.code, message: error?.message.trim() ?? '' }, children: [inspectWithNoCustomRetry(error, inspectOptions)], }); currentTest.failures = 1; @@ -227,13 +169,10 @@ module.exports = async function* junitReporter(source) { case 'test:diagnostic': { const parent = currentSuite?.children ?? roots; ArrayPrototypePush(parent, { - __proto__: null, - nesting: event.data.nesting, - comment: event.data.message, + __proto__: null, nesting: event.data.nesting, comment: event.data.message, }); break; - } - default: + } default: break; } } diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 6578290f714ceb..3d90d4368f0c21 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -19,11 +19,7 @@ const kDefaultIndent = ' '; // 4 spaces const kFrameStartRegExp = /^ {4}at /; const kLineBreakRegExp = /\n|\r\n/; const kDefaultTAPVersion = 13; -const inspectOptions = { - __proto__: null, - colors: false, - breakLength: Infinity, -}; +const inspectOptions = { __proto__: null, colors: false, breakLength: Infinity }; let testModule; // Lazy loaded due to circular dependency. function lazyLoadTest() { @@ -31,38 +27,18 @@ function lazyLoadTest() { return testModule; } -async function* tapReporter(source) { + +async function * tapReporter(source) { yield `TAP version ${kDefaultTAPVersion}\n`; for await (const { type, data } of source) { switch (type) { case 'test:fail': { - yield reportTest( - data.nesting, - data.testNumber, - 'not ok', - data.name, - data.skip, - data.todo, - data.expectFailure, - data.flakyRetriedCount, - ); - const location = data.file - ? `${data.file}:${data.line}:${data.column}` - : null; + yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.retryCount); + const location = data.file ? `${data.file}:${data.line}:${data.column}` : null; yield reportDetails(data.nesting, data.details, location); break; - } - case 'test:pass': - yield reportTest( - data.nesting, - data.testNumber, - 'ok', - data.name, - data.skip, - data.todo, - data.expectFailure, - data.flakyRetriedCount, - ); + } case 'test:pass': + yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.retryCount); yield reportDetails(data.nesting, data.details, null); break; case 'test:plan': @@ -73,42 +49,23 @@ async function* tapReporter(source) { break; case 'test:stderr': case 'test:stdout': { - const lines = RegExpPrototypeSymbolSplit( - kLineBreakRegExp, - data.message, - ); + const lines = RegExpPrototypeSymbolSplit(kLineBreakRegExp, data.message); for (let i = 0; i < lines.length; i++) { if (lines[i].length === 0) continue; yield `# ${tapEscape(lines[i])}\n`; } break; - } - case 'test:diagnostic': + } case 'test:diagnostic': yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`; break; case 'test:coverage': - yield getCoverageReport( - indent(data.nesting), - data.summary, - '# ', - '', - true, - ); + yield getCoverageReport(indent(data.nesting), data.summary, '# ', '', true); break; } } } -function reportTest( - nesting, - testNumber, - status, - name, - skip, - todo, - expectFailure, - flakyRetriedCount, -) { +function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, retryCount) { let line = `${indent(nesting)}${status} ${testNumber}`; if (name) { @@ -121,9 +78,9 @@ function reportTest( line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { line += ' # EXPECTED FAILURE'; - } else if (flakyRetriedCount !== undefined && flakyRetriedCount > 0) { - const retryText = flakyRetriedCount === 1 ? 're-try' : 're-tries'; - line += ` # FLAKY ${flakyRetriedCount} ${retryText}`; + } else if (retryCount !== undefined && retryCount > 0) { + const retryText = retryCount === 1 ? 're-try' : 're-tries'; + line += ` # FLAKY ${retryCount} ${retryText}`; } line += '\n'; @@ -159,6 +116,7 @@ function indent(nesting) { return value; } + // In certain places, # and \ need to be escaped as \# and \\. function tapEscape(input) { let result = StringPrototypeReplaceAll(input, '\b', '\\b'); @@ -322,12 +280,7 @@ function jsToYaml(indent, name, value, seen) { } function isAssertionLike(value) { - return ( - value && - typeof value === 'object' && - 'expected' in value && - 'actual' in value - ); + return value && typeof value === 'object' && 'expected' in value && 'actual' in value; } module.exports = tapReporter; diff --git a/lib/internal/test_runner/reporter/utils.js b/lib/internal/test_runner/reporter/utils.js index 1986adfb1572a8..990bcfaf55d693 100644 --- a/lib/internal/test_runner/reporter/utils.js +++ b/lib/internal/test_runner/reporter/utils.js @@ -17,7 +17,7 @@ const inspectOptions = { }; const reporterUnicodeSymbolMap = { - __proto__: null, + '__proto__': null, 'test:fail': '\u2716 ', 'test:pass': '\u2714 ', 'test:diagnostic': '\u2139 ', @@ -28,7 +28,7 @@ const reporterUnicodeSymbolMap = { }; const reporterColorMap = { - __proto__: null, + '__proto__': null, get 'test:fail'() { return colors.red; }, @@ -38,13 +38,13 @@ const reporterColorMap = { get 'test:diagnostic'() { return colors.blue; }, - get info() { + get 'info'() { return colors.blue; }, - get warn() { + get 'warn'() { return colors.yellow; }, - get error() { + get 'error'() { return colors.red; }, }; @@ -64,25 +64,15 @@ function formatError(error, indent) { RegExpPrototypeSymbolSplit( hardenRegExp(/\r?\n/), inspectWithNoCustomRetry(err, inspectOptions), - ), - `\n${indent} `, - ); + ), `\n${indent} `); return `\n${indent} ${message}\n`; } -function formatTestReport( - type, - data, - showErrorDetails = true, - prefix = '', - indent = '', -) { +function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') { let color = reporterColorMap[type] ?? colors.white; let symbol = reporterUnicodeSymbolMap[type] ?? ' '; - const { skip, todo, expectFailure, flakyRetriedCount } = data; - const duration_ms = data.details?.duration_ms - ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` - : ''; + const { skip, todo, expectFailure, retryCount } = data; + const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : ''; let title = `${data.name}${duration_ms}`; if (skip !== undefined) { @@ -97,15 +87,12 @@ function formatTestReport( } } else if (expectFailure !== undefined) { title += ` # EXPECTED FAILURE`; - } else if (flakyRetriedCount !== undefined && flakyRetriedCount > 0) { - const retryText = flakyRetriedCount === 1 ? 're-try' : 're-tries'; - title += ` # FLAKY ${flakyRetriedCount} ${retryText}`; + } else if (retryCount !== undefined && retryCount > 0) { + const retryText = retryCount === 1 ? 're-try' : 're-tries'; + title += ` # FLAKY ${retryCount} ${retryText}`; } - const err = - showErrorDetails && data.details?.error - ? formatError(data.details.error, indent) - : ''; + const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : ''; return `${prefix}${indent}${color}${symbol}${title}${colors.white}${err}`; } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 9b994ba7b714f8..9aa16961f165f1 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -32,16 +32,17 @@ const { SymbolDispose, } = primordials; const { getCallerLocation } = internalBinding('util'); -const { - exitCodes: { kGenericUserError }, -} = internalBinding('errors'); +const { exitCodes: { kGenericUserError } } = internalBinding('errors'); const { addAbortListener } = require('internal/events/abort_listener'); const { queueMicrotask } = require('internal/process/task_queues'); const { AsyncResource } = require('async_hooks'); const { AbortController } = require('internal/abort_controller'); const { AbortError, - codes: { ERR_INVALID_ARG_TYPE, ERR_TEST_FAILURE }, + codes: { + ERR_INVALID_ARG_TYPE, + ERR_TEST_FAILURE, + }, } = require('internal/errors'); const { MockTracker } = require('internal/test_runner/mock/mock'); const { TestsStream } = require('internal/test_runner/tests_stream'); @@ -65,7 +66,10 @@ const { validateOneOf, validateUint32, } = require('internal/validators'); -const { clearTimeout, setTimeout } = require('timers'); +const { + clearTimeout, + setTimeout, +} = require('timers'); const { TIMEOUT_MAX } = require('internal/timers'); const { fileURLToPath } = require('internal/url'); const { relative } = require('path'); @@ -87,10 +91,8 @@ const noop = FunctionPrototype; const kShouldAbort = Symbol('kShouldAbort'); const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']); const kUnwrapErrors = new SafeSet() - .add(kTestCodeFailure) - .add(kHookFailure) - .add('uncaughtException') - .add('unhandledRejection'); + .add(kTestCodeFailure).add(kHookFailure) + .add('uncaughtException').add('unhandledRejection'); let kResistStopPropagation; let assertObj; let findSourceMap; @@ -112,9 +114,7 @@ function lazyAssertObject(harness) { const { SnapshotManager } = require('internal/test_runner/snapshot'); assertObj = getAssertionMap(); - harness.snapshotManager = new SnapshotManager( - harness.config.updateSnapshots, - ); + harness.snapshotManager = new SnapshotManager(harness.config.updateSnapshots); if (!assertObj.has('snapshot')) { assertObj.set('snapshot', harness.snapshotManager.createAssert()); @@ -138,16 +138,12 @@ function stopTest(timeout, signal) { } else { timer = setTimeout(deferred.resolve, timeout); timer.unref(); - setOwnProperty( - deferred, - 'promise', - PromisePrototypeThen(deferred.promise, () => { - throw new ERR_TEST_FAILURE( - `test timed out after ${timeout}ms`, - kTestTimeoutFailure, - ); - }), - ); + setOwnProperty(deferred, 'promise', PromisePrototypeThen(deferred.promise, () => { + throw new ERR_TEST_FAILURE( + `test timed out after ${timeout}ms`, + kTestTimeoutFailure, + ); + })); disposeFunction = () => { abortListener[SymbolDispose](); @@ -160,21 +156,15 @@ function stopTest(timeout, signal) { } function testMatchesPattern(test, patterns) { - const matchesByNameOrParent = - ArrayPrototypeSome( - patterns, - (re) => RegExpPrototypeExec(re, test.name) !== null, - ) || - (test.parent && testMatchesPattern(test.parent, patterns)); + const matchesByNameOrParent = ArrayPrototypeSome(patterns, (re) => + RegExpPrototypeExec(re, test.name) !== null, + ) || (test.parent && testMatchesPattern(test.parent, patterns)); if (matchesByNameOrParent) return true; - const testNameWithAncestors = StringPrototypeTrim( - test.getTestNameWithAncestors(), - ); + const testNameWithAncestors = StringPrototypeTrim(test.getTestNameWithAncestors()); - return ArrayPrototypeSome( - patterns, - (re) => RegExpPrototypeExec(re, testNameWithAncestors) !== null, + return ArrayPrototypeSome(patterns, (re) => + RegExpPrototypeExec(re, testNameWithAncestors) !== null, ); } @@ -197,11 +187,7 @@ class TestPlan { validateNumber(wait, 'options.wait', 0, TIMEOUT_MAX); this.wait = wait; } else if (wait !== undefined) { - throw new ERR_INVALID_ARG_TYPE( - 'options.wait', - ['boolean', 'number'], - wait, - ); + throw new ERR_INVALID_ARG_TYPE('options.wait', ['boolean', 'number'], wait); } } @@ -264,6 +250,7 @@ class TestPlan { } } + class TestContext { #assert; #test; @@ -377,11 +364,7 @@ class TestContext { const subtest = this.#test.createSubtest( // eslint-disable-next-line no-use-before-define - Test, - name, - options, - fn, - overrides, + Test, name, options, fn, overrides, ); return subtest.start(); @@ -431,7 +414,10 @@ class TestContext { validateFunction(condition, 'condition'); validateObject(options, 'options'); - const { interval = 50, timeout = 1000 } = options; + const { + interval = 50, + timeout = 1000, + } = options; validateNumber(interval, 'options.interval', 0, TIMEOUT_MAX); validateNumber(timeout, 'options.timeout', 0, TIMEOUT_MAX); @@ -513,20 +499,7 @@ class Test extends AsyncResource { super('Test'); let { fn, name, parent } = options; - - const { - concurrency, - entryFile, - expectFailure, - flaky, - loc, - only, - timeout, - todo, - skip, - signal, - plan, - } = options; + const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options; if (typeof fn !== 'function') { fn = noop; @@ -563,10 +536,9 @@ class Test extends AsyncResource { this.entryFile = entryFile; this.testDisambiguator = new SafeMap(); } else { - const nesting = - parent.parent === null ? parent.nesting : parent.nesting + 1; - const { config, isFilteringByName, isFilteringByOnly } = - parent.root.harness; + const nesting = parent.parent === null ? parent.nesting : + parent.nesting + 1; + const { config, isFilteringByName, isFilteringByOnly } = parent.root.harness; this.root = parent.root; this.harness = null; @@ -583,11 +555,7 @@ class Test extends AsyncResource { if (isFilteringByName) { this.filteredByName = this.willBeFilteredByName(); if (!this.filteredByName) { - for ( - let t = this.parent; - t !== null && t.filteredByName; - t = t.parent - ) { + for (let t = this.parent; t !== null && t.filteredByName; t = t.parent) { t.filteredByName = false; } } @@ -601,11 +569,7 @@ class Test extends AsyncResource { this.parent.runOnlySubtests = true; if (this.parent === this.root || this.parent.startTime === null) { - for ( - let t = this.parent; - t !== null && !t.hasOnlyTests; - t = t.parent - ) { + for (let t = this.parent; t !== null && !t.hasOnlyTests; t = t.parent) { t.hasOnlyTests = true; } } @@ -627,8 +591,8 @@ class Test extends AsyncResource { case 'boolean': if (concurrency) { - this.concurrency = - parent === null ? MathMax(availableParallelism() - 1, 1) : Infinity; + this.concurrency = parent === null ? + MathMax(availableParallelism() - 1, 1) : Infinity; } else { this.concurrency = 1; } @@ -636,11 +600,7 @@ class Test extends AsyncResource { default: if (concurrency != null) - throw new ERR_INVALID_ARG_TYPE( - 'options.concurrency', - ['boolean', 'number'], - concurrency, - ); + throw new ERR_INVALID_ARG_TYPE('options.concurrency', ['boolean', 'number'], concurrency); } if (timeout != null && timeout !== Infinity) { @@ -664,14 +624,14 @@ class Test extends AsyncResource { validateAbortSignal(signal, 'options.signal'); if (signal) { - kResistStopPropagation ??= - require('internal/event_target').kResistStopPropagation; + kResistStopPropagation ??= require('internal/event_target').kResistStopPropagation; } - this.outerSignal?.addEventListener('abort', this.#abortHandler, { - __proto__: null, - [kResistStopPropagation]: true, - }); + this.outerSignal?.addEventListener( + 'abort', + this.#abortHandler, + { __proto__: null, [kResistStopPropagation]: true }, + ); this.fn = fn; this.mock = null; @@ -705,8 +665,8 @@ class Test extends AsyncResource { this.error = null; this.attempt = undefined; this.passedAttempt = undefined; - this.message = - typeof skip === 'string' ? skip : typeof todo === 'string' ? todo : null; + this.message = typeof skip === 'string' ? skip : + typeof todo === 'string' ? todo : null; this.activeSubtests = 0; this.pendingSubtests = []; this.readySubtests = new SafeMap(); @@ -760,24 +720,16 @@ class Test extends AsyncResource { this.root.testDisambiguator.set(testIdentifier, 1); } this.attempt = this.root.harness.previousRuns.length; - const previousAttempt = - this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]; + const previousAttempt = this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]; if (previousAttempt != null) { this.passedAttempt = previousAttempt.passed_on_attempt; this.fn = () => { for (let i = 0; i < (previousAttempt.children?.length ?? 0); i++) { const child = previousAttempt.children[i]; - this.createSubtest( - Test, - child.name, - { __proto__: null }, - noop, - { - __proto__: null, - loc: [child.line, child.column, child.file], - }, - noop, - ).start(); + this.createSubtest(Test, child.name, { __proto__: null }, noop, { + __proto__: null, + loc: [child.line, child.column, child.file], + }, noop).start(); } }; } @@ -795,16 +747,10 @@ class Test extends AsyncResource { return; } - if ( - this.root.harness.isFilteringByOnly && - !this.only && - !this.hasOnlyTests - ) { - if ( - this.parent.runOnlySubtests || - this.parent.hasOnlyTests || - this.only === false - ) { + if (this.root.harness.isFilteringByOnly && !this.only && !this.hasOnlyTests) { + if (this.parent.runOnlySubtests || + this.parent.hasOnlyTests || + this.only === false) { this.filtered = true; } } @@ -857,12 +803,7 @@ class Test extends AsyncResource { while (this.pendingSubtests.length > 0 && this.hasConcurrency()) { const deferred = ArrayPrototypeShift(this.pendingSubtests); const test = deferred.test; - test.reporter.dequeue( - test.nesting, - test.loc, - test.name, - this.reportedType, - ); + test.reporter.dequeue(test.nesting, test.loc, test.name, this.reportedType); await test.run(); deferred.resolve(); } @@ -875,10 +816,8 @@ class Test extends AsyncResource { addReadySubtest(subtest) { this.readySubtests.set(subtest.childNumber, subtest); - if ( - this.unfinishedSubtests.delete(subtest) && - this.unfinishedSubtests.size === 0 - ) { + if (this.unfinishedSubtests.delete(subtest) && + this.unfinishedSubtests.size === 0) { this.subtestsPromise.resolve(); } } @@ -942,14 +881,7 @@ class Test extends AsyncResource { } } - const test = new Factory({ - __proto__: null, - fn, - name, - parent, - ...options, - ...overrides, - }); + const test = new Factory({ __proto__: null, fn, name, parent, ...options, ...overrides }); if (parent.waitingOn === 0) { parent.waitingOn = test.childNumber; @@ -970,8 +902,7 @@ class Test extends AsyncResource { } #abortHandler = () => { - const error = - this.outerSignal?.reason || new AbortError('The test was aborted'); + const error = this.outerSignal?.reason || new AbortError('The test was aborted'); error.failureType = kAborted; this.#cancel(error); }; @@ -981,12 +912,11 @@ class Test extends AsyncResource { return; } - this.fail( - error || - new ERR_TEST_FAILURE( - 'test did not finish before its parent and was cancelled', - kCancelledByParent, - ), + this.fail(error || + new ERR_TEST_FAILURE( + 'test did not finish before its parent and was cancelled', + kCancelledByParent, + ), ); this.cancelled = true; this.abortController.abort(); @@ -1002,8 +932,7 @@ class Test extends AsyncResource { if (this.parent.hooks.afterEach.length > 0) { ArrayPrototypePushApply( - this.hooks.afterEach, - ArrayPrototypeSlice(this.parent.hooks.afterEach), + this.hooks.afterEach, ArrayPrototypeSlice(this.parent.hooks.afterEach), ); } } @@ -1027,12 +956,7 @@ class Test extends AsyncResource { // afterEach hooks for the current test should run in the order that they // are created. However, the current test's afterEach hooks should run // prior to any ancestor afterEach hooks. - ArrayPrototypeSplice( - this.hooks[name], - this.hooks.ownAfterEachCount, - 0, - hook, - ); + ArrayPrototypeSplice(this.hooks[name], this.hooks.ownAfterEachCount, 0, hook); this.hooks.ownAfterEachCount++; } else { ArrayPrototypePush(this.hooks[name], hook); @@ -1135,10 +1059,7 @@ class Test extends AsyncResource { } } } catch (err) { - const error = new ERR_TEST_FAILURE( - `failed running ${hook} hook`, - kHookFailure, - ); + const error = new ERR_TEST_FAILURE(`failed running ${hook} hook`, kHookFailure); error.cause = isTestFailureError(err) ? err.cause : err; throw error; } @@ -1207,29 +1128,30 @@ class Test extends AsyncResource { const promises = []; if (this.fn.length === runArgs.length - 1) { + // This test is using legacy Node.js error-first callbacks. const { promise, cb } = createDeferredCallback(); ArrayPrototypePush(runArgs, cb); const ret = ReflectApply(this.runInAsyncScope, this, runArgs); if (isPromise(ret)) { - this.fail( - new ERR_TEST_FAILURE( - 'passed a callback but also returned a Promise', - kCallbackAndPromisePresent, - ), - ); + this.fail(new ERR_TEST_FAILURE( + 'passed a callback but also returned a Promise', + kCallbackAndPromisePresent, + )); ArrayPrototypePush(promises, ret); } else { ArrayPrototypePush(promises, PromiseResolve(promise)); } } else { + // This test is synchronous or using Promises. const promise = ReflectApply(this.runInAsyncScope, this, runArgs); ArrayPrototypePush(promises, PromiseResolve(promise)); } ArrayPrototypePush(promises, stopPromise); + // Wait for the race to finish await SafePromiseRace(promises); this[kShouldAbort](); @@ -1240,6 +1162,8 @@ class Test extends AsyncResource { if (this.plan !== null) { const planPromise = this.plan?.check(); + // If the plan returns a promise, it means that it is waiting for more assertions to be made before + // continuing. if (planPromise) { await SafePromiseRace([planPromise, stopPromise]); } @@ -1267,16 +1191,8 @@ class Test extends AsyncResource { } else { this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure)); } - try { - await afterEach(); - } catch { - /* test is already failing, let's ignore the error */ - } - try { - await after(); - } catch { - /* Ignore error. */ - } + try { await afterEach(); } catch { /* test is already failing, let's ignore the error */ } + try { await after(); } catch { /* Ignore error. */ } } finally { stopPromise?.[SymbolDispose](); @@ -1307,21 +1223,15 @@ class Test extends AsyncResource { for (let i = 0; i < reporterScope.reporters.length; i++) { const { destination } = reporterScope.reporters[i]; - ArrayPrototypePush( - promises, - new Promise((resolve) => { - destination.on('unpipe', () => { - if ( - !destination.closed && - typeof destination.close === 'function' - ) { - destination.close(resolve); - } else { - resolve(); - } - }); - }), - ); + ArrayPrototypePush(promises, new Promise((resolve) => { + destination.on('unpipe', () => { + if (!destination.closed && typeof destination.close === 'function') { + destination.close(resolve); + } else { + resolve(); + } + }); + })); } this.harness.teardown(); @@ -1367,14 +1277,7 @@ class Test extends AsyncResource { const report = this.getReportDetails(); report.details.passed = this.passed; this.testNumber ||= ++this.parent.outputSubtestCount; - this.reporter.complete( - this.nesting, - this.loc, - this.testNumber, - this.name, - report.details, - report.directive, - ); + this.reporter.complete(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); this.parent.activeSubtests--; } @@ -1382,7 +1285,13 @@ class Test extends AsyncResource { this.parent.processReadySubtestRange(false); this.parent.processPendingSubtests(); } else if (!this.reported) { - const { diagnostics, harness, loc, nesting, reporter } = this; + const { + diagnostics, + harness, + loc, + nesting, + reporter, + } = this; this.reported = true; reporter.plan(nesting, loc, harness.counters.topLevel); @@ -1399,11 +1308,7 @@ class Test extends AsyncResource { reporter.diagnostic(nesting, loc, `suites ${harness.counters.suites}`); reporter.diagnostic(nesting, loc, `pass ${harness.counters.passed}`); reporter.diagnostic(nesting, loc, `fail ${harness.counters.failed}`); - reporter.diagnostic( - nesting, - loc, - `cancelled ${harness.counters.cancelled}`, - ); + reporter.diagnostic(nesting, loc, `cancelled ${harness.counters.cancelled}`); reporter.diagnostic(nesting, loc, `skipped ${harness.counters.skipped}`); reporter.diagnostic(nesting, loc, `todo ${harness.counters.todo}`); reporter.diagnostic(nesting, loc, `flaky ${harness.counters.flaky}`); @@ -1411,26 +1316,14 @@ class Test extends AsyncResource { if (coverage) { const coverages = [ - { - __proto__: null, - actual: coverage.totals.coveredLinePercent, - threshold: this.config.lineCoverage, - name: 'line', - }, - - { - __proto__: null, - actual: coverage.totals.coveredBranchPercent, - threshold: this.config.branchCoverage, - name: 'branch', - }, - - { - __proto__: null, - actual: coverage.totals.coveredFunctionPercent, - threshold: this.config.functionCoverage, - name: 'function', - }, + { __proto__: null, actual: coverage.totals.coveredLinePercent, + threshold: this.config.lineCoverage, name: 'line' }, + + { __proto__: null, actual: coverage.totals.coveredBranchPercent, + threshold: this.config.branchCoverage, name: 'branch' }, + + { __proto__: null, actual: coverage.totals.coveredFunctionPercent, + threshold: this.config.functionCoverage, name: 'function' }, ]; for (let i = 0; i < coverages.length; i++) { @@ -1438,12 +1331,7 @@ class Test extends AsyncResource { if (actual < threshold) { harness.success = false; process.exitCode = kGenericUserError; - reporter.diagnostic( - nesting, - loc, - `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`, - 'error', - ); + reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`, 'error'); } } @@ -1451,11 +1339,7 @@ class Test extends AsyncResource { } reporter.summary( - nesting, - loc?.file, - harness.success, - harness.counters, - duration, + nesting, loc?.file, harness.success, harness.counters, duration, ); if (harness.watching) { @@ -1469,11 +1353,10 @@ class Test extends AsyncResource { } isClearToSend() { - return ( - this.parent === null || - (this.parent.waitingOn === this.childNumber && - this.parent.isClearToSend()) - ); + return this.parent === null || + ( + this.parent.waitingOn === this.childNumber && this.parent.isClearToSend() + ); } finalize() { @@ -1492,10 +1375,8 @@ class Test extends AsyncResource { this.parent.waitingOn++; this.finished = true; - if ( - this.parent === this.root && - this.root.waitingOn > this.root.subtests.length - ) { + if (this.parent === this.root && + this.root.waitingOn > this.root.subtests.length) { // At this point all of the tests have finished running. However, there // might be ref'ed handles keeping the event loop alive. This gives the // global after() hook a chance to clean them up. The user may also @@ -1542,34 +1423,16 @@ class Test extends AsyncResource { report() { countCompletedTest(this); if (this.outputSubtestCount > 0) { - this.reporter.plan( - this.subtests[0].nesting, - this.loc, - this.outputSubtestCount, - ); + this.reporter.plan(this.subtests[0].nesting, this.loc, this.outputSubtestCount); } else { this.reportStarted(); } const report = this.getReportDetails(); if (this.passed) { - this.reporter.ok( - this.nesting, - this.loc, - this.testNumber, - this.name, - report.details, - report.directive, - ); + this.reporter.ok(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); } else { - this.reporter.fail( - this.nesting, - this.loc, - this.testNumber, - this.name, - report.details, - report.directive, - ); + this.reporter.fail(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); } for (let i = 0; i < this.diagnostics.length; i++) { @@ -1636,18 +1499,11 @@ class TestHook extends Test { } this.endTime ??= hrtime(); - parent.reporter.fail( - 0, - loc, - parent.subtests.length + 1, - loc.file, - { - __proto__: null, - duration_ms: this.duration(), - error, - }, - undefined, - ); + parent.reporter.fail(0, loc, parent.subtests.length + 1, loc.file, { + __proto__: null, + duration_ms: this.duration(), + error, + }, undefined); } } } @@ -1660,11 +1516,9 @@ class Suite extends Test { this.timeout = null; } - if ( - this.config.testNamePatterns !== null && - this.config.testSkipPatterns !== null && - !options.skip - ) { + if (this.config.testNamePatterns !== null && + this.config.testSkipPatterns !== null && + !options.skip) { this.fn = options.fn || this.fn; this.skipped = false; } @@ -1696,10 +1550,7 @@ class Suite extends Test { const hookArgs = this.getRunArgs(); let stopPromise; - const after = runOnce( - () => this.runHook('after', hookArgs), - kRunOnceOptions, - ); + const after = runOnce(() => this.runHook('after', hookArgs), kRunOnceOptions); try { this.parent.activeSubtests++; await this.buildSuite; @@ -1726,11 +1577,7 @@ class Suite extends Test { this.pass(); } catch (err) { - try { - await after(); - } catch { - /* suite is already failing */ - } + try { await after(); } catch { /* suite is already failing */ } if (isTestFailureError(err)) { this.fail(err); } else { diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js index 5e5ed10044165e..cdb293807ac8c4 100644 --- a/lib/internal/test_runner/tests_stream.js +++ b/lib/internal/test_runner/tests_stream.js @@ -92,7 +92,7 @@ class TestsStream extends Readable { } getFlaky(retriesTaken = undefined) { - return { __proto__: null, flakyRetriedCount: retriesTaken ?? 0 }; + return { __proto__: null, retryCount: retriesTaken ?? 0 }; } enqueue(nesting, loc, name, type) {