test_runner: add --test-randomize functionality

Author: pmarchini

doc/api/test.md

@@ -585,6 +585,32 @@ prevent shell expansion, which can reduce portability across systems.
 node --test "**/*.test.js" "**/*.spec.js"
 ```
 
+### Randomizing test file execution order
+
+The test runner can randomize the order of discovered test files to help detect
+order-dependent tests. Use `--test-randomize` to enable this mode.
+
+```bash
+node --test --test-randomize
+```
+
+When randomization is enabled, the test runner prints the seed used for the run
+as a diagnostic message:
+
+```text
+Randomized test order seed: 12345
+```
+
+Use `--test-random-seed=<number>` to replay the same order in a deterministic
+way. Supplying `--test-random-seed` also enables randomization, so
+`--test-randomize` is optional when a seed is provided:
+
+```bash
+node --test --test-randomize --test-random-seed=12345
+```
+
+`--test-randomize` and `--test-random-seed` are not supported with [`--watch`][] mode.
+
 Matching files are executed as test files.
 More information on the test file execution can be found
 in the [test runner execution model][] section.
@@ -625,6 +651,8 @@ test runner functionality:
 * `--test-reporter` - Reporting is managed by the parent process
 * `--test-reporter-destination` - Output destinations are controlled by the parent
 * `--experimental-config-file` - Config file paths are managed by the parent
+* `--test-randomize` - File randomization is managed by the parent process
+* `--test-random-seed` - File randomization seed is managed by the parent process
 
 All other Node.js options from command line arguments, environment variables,
 and configuration files are inherited by the child processes.
@@ -1531,6 +1559,13 @@ changes:
       that specifies the index of the shard to run. This option is _required_.
     * `total` {number} is a positive integer that specifies the total number
       of shards to split the test files to. This option is _required_.
+  * `randomize` {boolean} Randomize the execution order of test files.
+    This option is not supported with `watch: true`.
+    **Default:** `false`.
+  * `randomSeed` {number} Seed used when randomizing test file order. If this
+    option is set, runs can replay the same randomized file order
+    deterministically, and setting this option also enables randomization.
+    **Default:** `undefined`.
   * `rerunFailuresFilePath` {string} A file path where the test runner will
     store the state of the tests to allow rerunning only the failed tests on a next run.
     see \[Rerunning failed tests]\[] for more information.

doc/node-config-schema.json

@@ -910,6 +910,14 @@
           "type": "boolean",
           "description": "run tests with 'only' option set"
         },
+        "test-random-seed": {
+          "type": "number",
+          "description": "seed used to randomize test file execution order"
+        },
+        "test-randomize": {
+          "type": "boolean",
+          "description": "run test files in a random order"
+        },
         "test-reporter": {
           "oneOf": [
             {

lib/internal/test_runner/runner.js

@@ -56,6 +56,7 @@ const {
   validateObject,
   validateOneOf,
   validateInteger,
+  validateUint32,
   validateString,
   validateStringArray,
 } = require('internal/validators');
@@ -81,10 +82,12 @@ const {
 const { FastBuffer } = require('internal/buffer');
 
 const {
+  createRandomSeed,
   convertStringToRegExp,
   countCompletedTest,
   kDefaultPattern,
   parseCommandLine,
+  shuffleArrayWithSeed,
 } = require('internal/test_runner/utils');
 const { Glob } = require('internal/fs/glob');
 const { once } = require('events');
@@ -102,12 +105,14 @@ const kIsolatedProcessName = Symbol('kIsolatedProcessName');
 const kFilterArgs = [
   '--test',
   '--experimental-test-coverage',
+  '--test-randomize',
   '--watch',
   '--experimental-default-config-file',
 ];
 const kFilterArgValues = [
   '--test-reporter',
   '--test-reporter-destination',
+  '--test-random-seed',
   '--experimental-config-file',
 ];
 const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];
@@ -619,6 +624,8 @@ function run(options = kEmptyObject) {
     lineCoverage = 0,
     branchCoverage = 0,
     functionCoverage = 0,
+    randomize: suppliedRandomize,
+    randomSeed: suppliedRandomSeed,
     execArgv = [],
     argv = [],
     cwd = process.cwd(),
@@ -647,6 +654,37 @@ function run(options = kEmptyObject) {
   if (globPatterns != null) {
     validateArray(globPatterns, 'options.globPatterns');
   }
+  if (suppliedRandomize != null) {
+    validateBoolean(suppliedRandomize, 'options.randomize');
+  }
+  if (suppliedRandomSeed != null) {
+    validateUint32(suppliedRandomSeed, 'options.randomSeed');
+  }
+  let randomize = suppliedRandomize;
+  let randomSeed = suppliedRandomSeed;
+
+  if (randomSeed != null) {
+    randomize = true;
+  }
+  if (watch) {
+    if (randomSeed != null) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomSeed',
+        randomSeed,
+        'is not supported with watch mode',
+      );
+    }
+    if (randomize) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomize',
+        randomize,
+        'is not supported with watch mode',
+      );
+    }
+  }
+  if (randomize) {
+    randomSeed ??= createRandomSeed();
+  }
 
   validateString(cwd, 'options.cwd');
 
@@ -757,10 +795,16 @@ function run(options = kEmptyObject) {
     cwd,
     globalSetupPath,
   };
+
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns, cwd);
   const { isTestRunner } = globalOptions;
 
+  if (randomize) {
+    testFiles = shuffleArrayWithSeed(testFiles, randomSeed);
+    root.diagnostic(`Randomized test order seed: ${randomSeed}`);
+  }
+
   if (shard) {
     testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1);
   }
@@ -786,6 +830,8 @@ function run(options = kEmptyObject) {
     execArgv,
     rerunFailuresFilePath,
     env,
+    randomize,
+    randomSeed,
   };
 
   if (isolation === 'process') {

lib/internal/test_runner/utils.js

@@ -7,11 +7,14 @@ const {
   ArrayPrototypePop,
   ArrayPrototypePush,
   ArrayPrototypeReduce,
+  ArrayPrototypeSlice,
   ArrayPrototypeSome,
   JSONParse,
   MathFloor,
+  MathImul,
   MathMax,
   MathMin,
+  MathRandom,
   NumberParseInt,
   NumberPrototypeToFixed,
   ObjectGetOwnPropertyDescriptor,
@@ -45,6 +48,7 @@ const { compose } = require('stream');
 const {
   validateInteger,
   validateFunction,
+  validateUint32,
 } = require('internal/validators');
 const { validatePath } = require('internal/fs/utils');
 const { kEmptyObject } = require('internal/util');
@@ -58,6 +62,7 @@ const coverageColors = {
 
 const kMultipleCallbackInvocations = 'multipleCallbackInvocations';
 const kRegExpPattern = /^\/(.*)\/([a-z]*)$/;
+const kMaxRandomSeed = 0xFFFF_FFFF;
 
 const kPatterns = ['test', 'test/**/*', 'test-*', '*[._-]test'];
 const kFileExtensions = ['js', 'mjs', 'cjs'];
@@ -133,6 +138,54 @@ const kBuiltinReporters = new SafeMap([
 const kDefaultReporter = 'spec';
 const kDefaultDestination = 'stdout';
 
+/**
+ * Create a random uint32 seed.
+ * @returns {number}
+ */
+function createRandomSeed() {
+  return MathFloor(MathRandom() * (kMaxRandomSeed + 1));
+}
+
+/**
+ * Create a Mulberry32 pseudo-random number generator from a uint32 seed.
+ * @param {number} seed
+ * @returns {() => number}
+ */
+function createSeededGenerator(seed) {
+  let state = seed >>> 0;
+  return () => {
+    state = (state + 0x6D2B79F5) | 0;
+    let value = MathImul(state ^ state >>> 15, 1 | state);
+    value ^= value + MathImul(value ^ value >>> 7, 61 | value);
+    return ((value ^ value >>> 14) >>> 0) / 4_294_967_296;
+  };
+}
+
+/**
+ * Return a deterministically shuffled copy of an array.
+ * @template T
+ * @param {T[]} values
+ * @param {number} seed
+ * @returns {T[]}
+ */
+function shuffleArrayWithSeed(values, seed) {
+  if (values.length < 2) {
+    return values;
+  }
+
+  const randomized = ArrayPrototypeSlice(values);
+  const random = createSeededGenerator(seed);
+
+  for (let i = randomized.length - 1; i > 0; i--) {
+    const j = MathFloor(random() * (i + 1));
+    const tmp = randomized[i];
+    randomized[i] = randomized[j];
+    randomized[j] = tmp;
+  }
+
+  return randomized;
+}
+
 function tryBuiltinReporter(name) {
   const builtinPath = kBuiltinReporters.get(name);
 
@@ -217,6 +270,10 @@ function parseCommandLine() {
   const updateSnapshots = getOptionValue('--test-update-snapshots');
   const watch = getOptionValue('--watch');
   const timeout = getOptionValue('--test-timeout') || Infinity;
+  let randomize = getOptionValue('--test-randomize');
+  const hasRandomSeedOption = getOptionValue('[has_test_random_seed]');
+  const randomSeedOption = getOptionValue('--test-random-seed');
+  let randomSeed;
   const rerunFailuresFilePath = getOptionValue('--test-rerun-failures');
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
@@ -328,6 +385,12 @@ function parseCommandLine() {
     validatePath(rerunFailuresFilePath, '--test-rerun-failures');
   }
 
+  if (hasRandomSeedOption) {
+    validateUint32(randomSeedOption, '--test-random-seed');
+    randomSeed = randomSeedOption;
+    randomize = true;
+  }
+
   const setup = reporterScope.bind(async (rootReporter) => {
     const reportersMap = await getReportersMap(reporters, destinations);
     for (let i = 0; i < reportersMap.length; i++) {
@@ -362,6 +425,8 @@ function parseCommandLine() {
     timeout,
     updateSnapshots,
     watch,
+    randomize,
+    randomSeed,
     rerunFailuresFilePath,
   };
 
@@ -649,10 +714,14 @@ async function setupGlobalSetupTeardownFunctions(globalSetupPath, cwd) {
 module.exports = {
   convertStringToRegExp,
   countCompletedTest,
+  createRandomSeed,
+  createSeededGenerator,
   createDeferredCallback,
   isTestFailureError,
   kDefaultPattern,
+  kMaxRandomSeed,
   parseCommandLine,
+  shuffleArrayWithSeed,
   reporterScope,
   shouldColorizeTestFiles,
   getCoverageReport,

src/node_options.cc

@@ -987,6 +987,20 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             &EnvironmentOptions::test_global_setup_path,
             kAllowedInEnvvar,
             OptionNamespaces::kTestRunnerNamespace);
+  AddOption("--test-randomize",
+            "run test files in a random order",
+            &EnvironmentOptions::test_randomize,
+            kDisallowedInEnvvar,
+            false,
+            OptionNamespaces::kTestRunnerNamespace);
+  AddOption(
+      "[has_test_random_seed]", "", &EnvironmentOptions::has_test_random_seed);
+  AddOption("--test-random-seed",
+            "seed used to randomize test file execution order",
+            &EnvironmentOptions::test_random_seed,
+            kDisallowedInEnvvar,
+            OptionNamespaces::kTestRunnerNamespace);
+  Implies("--test-random-seed", "[has_test_random_seed]");
   AddOption("--test-rerun-failures",
             "specifies the path to the rerun state file",
             &EnvironmentOptions::test_rerun_failures_path,

src/node_options.h

@@ -204,6 +204,9 @@ class EnvironmentOptions : public Options {
   std::string test_rerun_failures_path;
   std::vector<std::string> test_reporter_destination;
   std::string test_global_setup_path;
+  bool test_randomize = false;
+  bool has_test_random_seed = false;
+  uint64_t test_random_seed = 0;
   bool test_only = false;
   bool test_udp_no_try_send = false;
   std::string test_isolation = "process";

test/fixtures/test-runner/output/randomize_output_cli.js

@@ -0,0 +1,16 @@
+'use strict';
+require('../../../common');
+const fixtures = require('../../../common/fixtures');
+const spawn = require('node:child_process').spawn;
+
+spawn(
+  process.execPath,
+  [
+    '--no-warnings',
+    '--test-reporter', 'tap',
+    '--test-random-seed=12345',
+    '--test',
+    fixtures.path('test-runner/shards/*.cjs'),
+  ],
+  { stdio: 'inherit' },
+);