Revert changes to default args; add a --no-randomize option instead

Author: mhsmith

Android/android.py

@@ -737,8 +737,10 @@ def ci(context):
             # Prove the package is self-contained by using it to run the tests.
             shutil.unpack_archive(package_path, temp_dir)
 
+            # Randomization is disabled because order-dependent failures are
+            # much less likely to pass on a rerun in single-process mode.
             launcher_args = ["--managed", "maxVersion", "-v"]
-            test_args = ["--fast-ci", "--single-process"]
+            test_args = ["--fast-ci", "--single-process", "--no-randomize"]
             run(
                 ["./android.py", "test", *launcher_args, "--", *test_args],
                 cwd=temp_dir

Lib/test/libregrtest/cmdline.py

@@ -155,7 +155,7 @@ def __init__(self, **kwargs) -> None:
         self.list_cases = False
         self.list_tests = False
         self.single = False
-        self.randomize = False
+        self.randomize = None
         self.fromfile = None
         self.fail_env_changed = False
         self.use_resources: list[str] = []
@@ -271,6 +271,9 @@ def _create_parser():
     group = parser.add_argument_group('Selecting tests')
     group.add_argument('-r', '--randomize', action='store_true',
                        help='randomize test execution order.' + more_details)
+    group.add_argument('--no-randomize', dest='randomize', action='store_false',
+                       help='do not randomize test execution order, even if '
+                       'it would be implied by another option')
     group.add_argument('--prioritize', metavar='TEST1,TEST2,...',
                        action='append', type=priority_list,
                        help='select these tests first, even if the order is'
@@ -453,13 +456,14 @@ def _parse_args(args, **kwargs):
     # Continuous Integration (CI): common options for fast/slow CI modes
     if ns.slow_ci or ns.fast_ci:
         # Similar to options:
-        #   -j0 --fail-env-changed --rerun --fail-rerun --slowest --verbose3
+        #   -j0 --randomize --fail-env-changed --rerun --slowest --verbose3
         if ns.use_mp is None:
             ns.use_mp = 0
+        if ns.randomize is None:
+            ns.randomize = True
         ns.fail_env_changed = True
         if ns.python is None:
             ns.rerun = True
-            ns.fail_rerun = True
         ns.print_slow = True
         ns.verbose3 = True
     else:
@@ -537,7 +541,7 @@ def _parse_args(args, **kwargs):
                         ns.use_resources.remove(r)
                 elif r not in ns.use_resources:
                     ns.use_resources.append(r)
-    if ns.random_seed is not None:
+    if ns.random_seed is not None and ns.randomize is None:
         ns.randomize = True
     if ns.verbose:
         ns.header = True

Lib/test/libregrtest/main.py

@@ -128,7 +128,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
         self._tmp_dir: StrPath | None = ns.tempdir
 
         # Randomize
-        self.randomize: bool = ns.randomize
+        self.randomize: bool = bool(ns.randomize)
         if ('SOURCE_DATE_EPOCH' in os.environ
             # don't use the variable if empty
             and os.environ['SOURCE_DATE_EPOCH']

Lib/test/test_regrtest.py

@@ -182,13 +182,33 @@ def test_randomize(self):
             self.assertTrue(regrtest.randomize)
             self.assertIsInstance(regrtest.random_seed, int)
 
+    def test_no_randomize(self):
+        ns = self.parse_args([])
+        self.assertIsNone(ns.randomize)
+
+        ns = self.parse_args(["--randomize"])
+        self.assertIs(ns.randomize, True)
+
+        ns = self.parse_args(["--no-randomize"])
+        self.assertIs(ns.randomize, False)
+
+        ns = self.parse_args(["--randomize", "--no-randomize"])
+        self.assertIs(ns.randomize, False)
+
+        ns = self.parse_args(["--no-randomize", "--randomize"])
+        self.assertIs(ns.randomize, True)
+
     def test_randseed(self):
         ns = self.parse_args(['--randseed', '12345'])
         self.assertEqual(ns.random_seed, 12345)
         self.assertTrue(ns.randomize)
         self.checkError(['--randseed'], 'expected one argument')
         self.checkError(['--randseed', 'foo'], 'invalid int value')
 
+        ns = self.parse_args(['--randseed', '12345', '--no-randomize'])
+        self.assertEqual(ns.random_seed, 12345)
+        self.assertFalse(ns.randomize)
+
     def test_fromfile(self):
         for opt in '-f', '--fromfile':
             with self.subTest(opt=opt):
@@ -428,11 +448,11 @@ def create_regrtest(self, args):
 
         return regrtest
 
-    def check_ci_mode(self, args, use_resources, rerun=True):
+    def check_ci_mode(self, args, use_resources, *, rerun=True, randomize=True):
         regrtest = self.create_regrtest(args)
         self.assertEqual(regrtest.num_workers, -1)
         self.assertEqual(regrtest.want_rerun, rerun)
-        self.assertEqual(regrtest.fail_rerun, rerun)
+        self.assertEqual(regrtest.randomize, randomize)
         self.assertIsInstance(regrtest.random_seed, int)
         self.assertTrue(regrtest.fail_env_changed)
         self.assertTrue(regrtest.print_slowest)
@@ -469,6 +489,15 @@ def test_slow_ci(self):
         regrtest = self.check_ci_mode(args, use_resources)
         self.assertEqual(regrtest.timeout, 20 * 60)
 
+    def test_ci_no_randomize(self):
+        all_resources = set(cmdline.ALL_RESOURCES)
+        self.check_ci_mode(
+            ["--slow-ci", "--no-randomize"], all_resources, randomize=False
+        )
+        self.check_ci_mode(
+            ["--fast-ci", "--no-randomize"], all_resources - {'cpu'}, randomize=False
+        )
+
     def test_dont_add_python_opts(self):
         args = ['--dont-add-python-opts']
         ns = cmdline._parse_args(args)