test_runner: add flaky option to retry on failure

doc/api/test.md

@@ -1868,6 +1868,16 @@ added:
 Shorthand for marking a suite as `TODO`. This is the same as
 [`suite([name], { todo: true }[, fn])`][suite options].
 
+## `suite.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`suite([name], { flaky: true }[, fn])`][suite options].
+
 ## `suite.only([name][, options][, fn])`
 
 <!-- YAML
@@ -1939,6 +1949,18 @@ changes:
   * `todo` {boolean|string} If truthy, the test marked as `TODO`. If a string
     is provided, that string is displayed in the test results as the reason why
     the test is `TODO`. **Default:** `false`.
+  * `flaky` {boolean|number} If `true`, the test (or, for a suite, each of its
+    test-cases) is retried until it passes, up to a default of 20 retries. If a
+    positive integer is provided, it is retried up to that many times. A
+    non-positive or non-integer value throws. Each retry re-runs the test's
+    `beforeEach` and `afterEach` hooks. A test-case's own value overrides one
+    inherited from its suite; `flaky: false` opts out. Retries are intended for
+    tests that fail intermittently due to transient conditions; be aware that
+    non-idempotent state may leak between attempts. A test that always fails
+    exhausts its full retry budget, running its body once plus once per retry
+    (for example 21 times with `flaky: true`: 1 initial run + 20 retries), so
+    prefer a small explicit count for tests that fail only occasionally.
+    **Default:** `false`.
   * `timeout` {number} A number of milliseconds the test will fail after.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `Infinity`.
@@ -2000,6 +2022,16 @@ same as [`test([name], { todo: true }[, fn])`][it options].
 Shorthand for marking a test as `only`,
 same as [`test([name], { only: true }[, fn])`][it options].
 
+## `test.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`test([name], { flaky: true }[, fn])`][it options].
+
 ## `describe([name][, options][, fn])`
 
 Alias for [`suite()`][].
@@ -2027,6 +2059,16 @@ added:
 Shorthand for marking a suite as `only`. This is the same as
 [`describe([name], { only: true }[, fn])`][describe options].
 
+## `describe.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a suite as flaky. This is the same as
+[`describe([name], { flaky: true }[, fn])`][describe options].
+
 ## `it([name][, options][, fn])`
 
 <!-- YAML
@@ -2066,6 +2108,16 @@ added:
 Shorthand for marking a test as `only`,
 same as [`it([name], { only: true }[, fn])`][it options].
 
+## `it.flaky([name][, options][, fn])`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Shorthand for marking a test as flaky,
+same as [`it([name], { flaky: true }[, fn])`][it options].
+
 ## `before([fn][, options])`
 
 <!-- YAML
@@ -3539,6 +3591,9 @@ Emitted when code coverage is enabled and all tests have completed.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries performed for a test
+    marked `flaky` (`0` if it passed on the first attempt). `undefined` for
+    tests that are not flaky.
 
 Emitted when a test completes its execution.
 This event is not emitted in the same order as the tests are
@@ -3647,6 +3702,9 @@ Emitted when a test is enqueued for execution.
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries performed for a test
+    marked `flaky` (`0` if it passed on the first attempt). `undefined` for
+    tests that are not flaky.
 
 Emitted when a test fails.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -3712,6 +3770,9 @@ since the parent runner only knows about file-level tests. When using
   * `testNumber` {number} The ordinal number of the test.
   * `todo` {string|boolean|undefined} Present if [`context.todo`][] is called
   * `skip` {string|boolean|undefined} Present if [`context.skip`][] is called
+  * `retryCount` {number|undefined} The number of retries performed for a test
+    marked `flaky` (`0` if it passed on the first attempt). `undefined` for
+    tests that are not flaky.
 
 Emitted when a test passes.
 This event is guaranteed to be emitted in the same order as the tests are
@@ -4527,6 +4588,18 @@ changes:
   * `todo` {boolean|string} If truthy, the test marked as `TODO`. If a string
     is provided, that string is displayed in the test results as the reason why
     the test is `TODO`. **Default:** `false`.
+  * `flaky` {boolean|number} If `true`, the test (or, for a suite, each of its
+    test-cases) is retried until it passes, up to a default of 20 retries. If a
+    positive integer is provided, it is retried up to that many times. A
+    non-positive or non-integer value throws. Each retry re-runs the test's
+    `beforeEach` and `afterEach` hooks. A test-case's own value overrides one
+    inherited from its suite; `flaky: false` opts out. Retries are intended for
+    tests that fail intermittently due to transient conditions; be aware that
+    non-idempotent state may leak between attempts. A test that always fails
+    exhausts its full retry budget, running its body once plus once per retry
+    (for example 21 times with `flaky: true`: 1 initial run + 20 retries), so
+    prefer a small explicit count for tests that fail only occasionally.
+    **Default:** `false`.
   * `timeout` {number} A number of milliseconds the test will fail after.
     If unspecified, subtests inherit this value from their parent.
     **Default:** `Infinity`.

lib/internal/test_runner/harness.js

@@ -63,6 +63,7 @@ function createTestTree(rootTestOptions, globalOptions) {
         failed: 0,
         passed: 0,
         cancelled: 0,
+        flaky: 0,
         skipped: 0,
         todo: 0,
         topLevel: 0,
@@ -406,7 +407,7 @@ function runInParentContext(Factory) {
 
     return run(name, options, fn, overrides);
   };
-  ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
+  ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => {
     test[keyword] = (name, options, fn) => {
       const overrides = {
         __proto__: null,

lib/internal/test_runner/reporter/dot.js

@@ -12,7 +12,11 @@ module.exports = async function* dot(source) {
   const failedTests = [];
   for await (const { type, data } of source) {
     if (type === 'test:pass') {
-      yield `${colors.green}.${colors.reset}`;
+      // A flaky test that needed retries is shown in yellow to distinguish it
+      // from a clean pass, without using 'F' (which reads as a failure).
+      yield data.retryCount > 0 ?
+        `${colors.yellow}.${colors.reset}` :
+        `${colors.green}.${colors.reset}`;
     }
     if (type === 'test:fail') {
       yield `${colors.red}X${colors.reset}`;

lib/internal/test_runner/reporter/junit.js

@@ -13,6 +13,7 @@ const {
 } = primordials;
 
 const { inspectWithNoCustomRetry } = require('internal/errors');
+const { formatRetryCount } = require('internal/test_runner/reporter/utils');
 const { hostname } = require('os');
 
 const inspectOptions = { __proto__: null, colors: false, breakLength: Infinity };
@@ -131,6 +132,16 @@ module.exports = async function* junitReporter(source) {
               attrs: { __proto__: null, type: 'todo', message: event.data.todo },
             });
           }
+          if (event.data.retryCount > 0) {
+            ArrayPrototypePush(currentTest.children, {
+              __proto__: null, nesting: event.data.nesting + 1, tag: 'properties',
+              attrs: { __proto__: null },
+              children: [{
+                __proto__: null, nesting: event.data.nesting + 2, tag: 'property',
+                attrs: { __proto__: null, name: 'flaky', value: formatRetryCount(event.data.retryCount) },
+              }],
+            });
+          }
           if (event.type === 'test:fail') {
             const error = event.data.details?.error;
             currentTest.children.push({

lib/internal/test_runner/reporter/tap.js

@@ -15,6 +15,7 @@ const {
 const { inspectWithNoCustomRetry } = require('internal/errors');
 const { isError, kEmptyObject } = require('internal/util');
 const { getCoverageReport } = require('internal/test_runner/utils');
+const { flakyDirective } = require('internal/test_runner/reporter/utils');
 const kDefaultIndent = '    '; // 4 spaces
 const kFrameStartRegExp = /^ {4}at /;
 const kLineBreakRegExp = /\n|\r\n/;
@@ -33,12 +34,14 @@ async function * tapReporter(source) {
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
-        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name,
+                         data.skip, data.todo, data.expectFailure, data.retryCount);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
         break;
       } case 'test:pass':
-        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
+        yield reportTest(data.nesting, data.testNumber, 'ok', data.name,
+                         data.skip, data.todo, data.expectFailure, data.retryCount);
         yield reportDetails(data.nesting, data.details, null);
         break;
       case 'test:plan':
@@ -75,7 +78,7 @@ async function * tapReporter(source) {
   }
 }
 
-function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
+function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, retryCount) {
   let line = `${indent(nesting)}${status} ${testNumber}`;
 
   if (name) {
@@ -88,6 +91,8 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   } else if (expectFailure !== undefined) {
     line += ` # EXPECTED FAILURE${typeof expectFailure === 'string' ? ` ${tapEscape(expectFailure)}` : ''}`;
+  } else if (retryCount > 0) {
+    line += flakyDirective(retryCount, status !== 'ok');
   }
 
   line += '\n';

lib/internal/test_runner/reporter/utils.js

@@ -68,10 +68,20 @@ function formatError(error, indent) {
   return `\n${indent}  ${message}\n`;
 }
 
+function formatRetryCount(retryCount) {
+  return `${retryCount} ${retryCount === 1 ? 're-try' : 're-tries'}`;
+}
+
+function flakyDirective(retryCount, failed) {
+  return failed ?
+    ` # FLAKY failed after ${formatRetryCount(retryCount)}` :
+    ` # FLAKY ${formatRetryCount(retryCount)}`;
+}
+
 function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') {
   let color = reporterColorMap[type] ?? colors.white;
   let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
-  const { skip, todo, expectFailure } = data;
+  const { skip, todo, expectFailure, retryCount } = data;
   const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
   const replayed = data.details?.passed_on_attempt !== undefined ?
     ` ${colors.gray}(passed on attempt ${data.details.passed_on_attempt})${colors.white}` :
@@ -90,6 +100,8 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde
     }
   } else if (expectFailure !== undefined) {
     title += ` # EXPECTED FAILURE`;
+  } else if (retryCount > 0) {
+    title += flakyDirective(retryCount, type === 'test:fail');
   }
 
   const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : '';
@@ -101,6 +113,8 @@ module.exports = {
   __proto__: null,
   reporterUnicodeSymbolMap,
   reporterColorMap,
+  flakyDirective,
+  formatRetryCount,
   formatTestReport,
   indent,
 };

lib/internal/test_runner/runner.js

@@ -313,7 +313,14 @@ class FileTest extends Test {
         skipped: item.data.skip !== undefined,
         isTodo: item.data.todo !== undefined,
         passed: item.type === 'test:pass',
-        cancelled: kCanceledTests.has(item.data.details?.error?.failureType),
+        // A retried timeout that exhausted (retryCount > 0) is a failure, not a
+        // cancellation only an un-retried timeout/abort stays cancelled.
+        cancelled: kCanceledTests.has(item.data.details?.error?.failureType) &&
+          !(item.data.retryCount > 0),
+        // retryCount is present (even 0) only for flaky-marked tests, so its
+        // presence is what marks this reconstructed test as flaky for the parent
+        // counter across the process-isolation IPC boundary.
+        isFlaky: item.data.retryCount !== undefined,
         nesting: item.data.nesting,
         reportedType: item.data.details?.type,
       }, this.root.harness);