feat: first commit flaky

doc/api/test.md

@@ -275,6 +275,51 @@ it.todo('should do the thing', { expectFailure: true }, () => {
 });
 ```
 
+## Flaky tests
+
+This flag causes a test or suite to be re-run a number of times until it
+either passes or has not passed after the final re-try.
+
+When `flaky` is `true`, the test harness re-tries the test up to the default
+number of times (20), inclusive.
+
+When `flaky` is a positive integer, the test harness re-tries the test up to
+the specified number of times, inclusive.
+
+When `flaky` is falsy (the default), the test harness does not re-try the test.
+
+When both a suite and an included test specify the `flaky` flag, the
+test's `flaky` value wins.
+
+```js
+it.flaky('should do something', () => {
+  // This test will be retried up to 20 times if it fails
+});
+
+it('may take several times', { flaky: true }, () => {
+  // Also retries up to 20 times
+});
+
+it('may also take several times', { flaky: 5 }, () => {
+  // Retries up to 5 times
+});
+
+describe.flaky('flaky suite', () => {
+  it('inherits flaky from suite', () => {
+    // Retried up to 20 times (inherited from suite)
+  });
+
+  it('not flaky', { flaky: false }, () => {
+    // Not retried, overrides suite setting
+  });
+});
+```
+
+When a test marked `flaky` passes after retries, the number of re-tries taken
+is reported with that test.
+
+`skip` and `todo` take precedence over `flaky`.
+
 ## `describe()` and `it()` aliases
 
 Suites and tests can also be written using the `describe()` and `it()`
@@ -1649,6 +1694,16 @@ added:
 Shorthand for marking a suite as `only`. This is the same as
 [`suite([name], { only: true }[, fn])`][suite options].
 
+## `suite.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`suite([name], { flaky: true }[, fn])`][suite options].
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -1684,6 +1739,11 @@ changes:
     thread. If `false`, only one test runs at a time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `false`.
+  * `flaky` {boolean|number} If truthy, the test is re-tried up to the
+    specified number of times (or `20` if `true`) until it passes. If the test
+    passes after retries, the number of retries taken is reported. When both a
+    suite and an included test specify the `flaky` flag, the test's value wins.
+    **Default:** `false`.
   * `only` {boolean} If truthy, and the test context is configured to run
     `only` tests, then this test will be run. Otherwise, the test is skipped.
     **Default:** `false`.
@@ -1755,6 +1815,16 @@ same as [`test([name], { todo: true }[, fn])`][it options].
 Shorthand for marking a test as `only`,
 same as [`test([name], { only: true }[, fn])`][it options].
 
+## `test.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`test([name], { flaky: true }[, fn])`][it options].
+
 ## `describe([name][, options][, fn])`
 
 Alias for [`suite()`][].
@@ -1782,6 +1852,16 @@ added:
 Shorthand for marking a suite as `only`. This is the same as
 [`describe([name], { only: true }[, fn])`][describe options].
 
+## `describe.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`describe([name], { flaky: true }[, fn])`][describe options].
+
 ## `it([name][, options][, fn])`
 
 <!-- YAML
@@ -1821,6 +1901,16 @@ added:
 Shorthand for marking a test as `only`,
 same as [`it([name], { only: true }[, fn])`][it options].
 
+## `it.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`it([name], { flaky: true }[, fn])`][it options].
+
 ## `before([fn][, options])`
 
 <!-- YAML
@@ -3342,6 +3432,8 @@ Emitted when a test is enqueued for execution.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries taken for a
+    flaky test. Present when a test is marked as flaky.
 
 Emitted when a test fails.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -3370,6 +3462,8 @@ The corresponding execution ordered event is `'test:complete'`.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries taken for a
+    flaky test. Present when a test is marked as flaky and passed after retries.
 
 Emitted when a test passes.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -3983,6 +4077,9 @@ changes:
     If `false`, it would only run one test at a time.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `null`.
+  * `flaky` {boolean|number} If truthy, the test is re-tried up to the
+    specified number of times (or `20` if `true`) until it passes.
+    **Default:** `false`.
   * `only` {boolean} If truthy, and the test context is configured to run
     `only` tests, then this test will be run. Otherwise, the test is skipped.
     **Default:** `false`.

lib/internal/test_runner/harness.js

@@ -61,6 +61,7 @@ function createTestTree(rootTestOptions, globalOptions) {
         failed: 0,
         passed: 0,
         cancelled: 0,
+        flaky: 0,
         skipped: 0,
         todo: 0,
         topLevel: 0,
@@ -377,7 +378,7 @@ function runInParentContext(Factory) {
 
     return run(name, options, fn, overrides);
   };
-  ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
+  ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => {
     test[keyword] = (name, options, fn) => {
       const overrides = {
         __proto__: null,

lib/internal/test_runner/reporter/dot.js

@@ -12,7 +12,11 @@ module.exports = async function* dot(source) {
   const failedTests = [];
   for await (const { type, data } of source) {
     if (type === 'test:pass') {
-      yield `${colors.green}.${colors.reset}`;
+      if (data.retryCount > 0) {
+        yield `${colors.yellow}F${colors.reset}`;
+      } else {
+        yield `${colors.green}.${colors.reset}`;
+      }
     }
     if (type === 'test:fail') {
       yield `${colors.red}X${colors.reset}`;

lib/internal/test_runner/reporter/junit.js

@@ -131,6 +131,26 @@ module.exports = async function* junitReporter(source) {
               attrs: { __proto__: null, type: 'todo', message: event.data.todo },
             });
           }
+          if (event.data.retryCount > 0) {
+            ArrayPrototypePush(currentTest.children, {
+              __proto__: null,
+              nesting: event.data.nesting + 1,
+              tag: 'properties',
+              attrs: { __proto__: null },
+              children: [
+                {
+                  __proto__: null,
+                  nesting: event.data.nesting + 2,
+                  tag: 'property',
+                  attrs: {
+                    __proto__: null,
+                    name: 'flaky',
+                    value: `${event.data.retryCount} retries`,
+                  },
+                },
+              ],
+            });
+          }
           if (event.type === 'test:fail') {
             const error = event.data.details?.error;
             currentTest.children.push({

lib/internal/test_runner/reporter/tap.js

@@ -33,12 +33,12 @@ async function * tapReporter(source) {
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
-        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.retryCount);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
         break;
       } case 'test:pass':
-        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.retryCount);
         yield reportDetails(data.nesting, data.details, null);
         break;
       case 'test:plan':
@@ -65,7 +65,7 @@ async function * tapReporter(source) {
   }
 }
 
-function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
+function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, retryCount) {
   let line = `${indent(nesting)}${status} ${testNumber}`;
 
   if (name) {
@@ -78,6 +78,9 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
     line += ' # EXPECTED FAILURE';
+  } else if (retryCount !== undefined && retryCount > 0) {
+    const retryText = retryCount === 1 ? 're-try' : 're-tries';
+    line += ` # FLAKY ${retryCount} ${retryText}`;
   }
 
   line += '\n';

lib/internal/test_runner/reporter/utils.js

@@ -71,7 +71,7 @@ function formatError(error, indent) {
 function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') {
   let color = reporterColorMap[type] ?? colors.white;
   let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
-  const { skip, todo, expectFailure } = data;
+  const { skip, todo, expectFailure, retryCount } = data;
   const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
   let title = `${data.name}${duration_ms}`;
 
@@ -87,6 +87,9 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde
     }
   } else if (expectFailure !== undefined) {
     title += ` # EXPECTED FAILURE`;
+  } else if (retryCount !== undefined && retryCount > 0) {
+    const retryText = retryCount === 1 ? 're-try' : 're-tries';
+    title += ` # FLAKY ${retryCount} ${retryText}`;
   }
 
   const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : '';