gh-110171: libregrtest always sets random.seed unless --no-use-randseed is provided

Author: sobolevn

Lib/test/libregrtest/cmdline.py

@@ -27,8 +27,11 @@
 Additional option details:
 
 -r randomizes test execution order. You can use --randseed=int to provide an
-int seed value for the randomizer; this is useful for reproducing troublesome
-test orders.
+int seed value for the randomizer. The randseed value will be used
+to set seeds for all random usages in tests
+(including randomizing the tests order if -r is set).
+By default we always set random seed, but do not randomize test order.
+Use --no-use-randseed to disable random seeding.
 
 -s On the first invocation of regrtest using -s, the first test file found
 or the first test file given on the command line is run, and the name of
@@ -156,6 +159,7 @@ def __init__(self, **kwargs) -> None:
         self.list_tests = False
         self.single = False
         self.randomize = False
+        self.use_random_seed = True
         self.fromfile = None
         self.fail_env_changed = False
         self.use_resources = None
@@ -229,6 +233,13 @@ def _create_parser():
                             more_details)
     group.add_argument('-p', '--python', metavar='PYTHON',
                        help='Command to run Python test subprocesses with.')
+    group.add_argument('--randseed', metavar='SEED',
+                       dest='random_seed', type=int,
+                       help='pass a global random seed')
+    group.add_argument('--use-randseed',
+                       dest='use_random_seed',
+                       action=argparse.BooleanOptionalAction,
+                       help='control if random should be seeded')
 
     group = parser.add_argument_group('Verbosity')
     group.add_argument('-v', '--verbose', action='count',
@@ -249,10 +260,6 @@ def _create_parser():
     group = parser.add_argument_group('Selecting tests')
     group.add_argument('-r', '--randomize', action='store_true',
                        help='randomize test execution order.' + more_details)
-    group.add_argument('--randseed', metavar='SEED',
-                       dest='random_seed', type=int,
-                       help='pass a random seed to reproduce a previous '
-                            'random run')
     group.add_argument('-f', '--fromfile', metavar='FILE',
                        help='read names of tests to run from a file.' +
                             more_details)
@@ -483,6 +490,11 @@ def _parse_args(args, **kwargs):
                         ns.use_resources.remove(r)
                 elif r not in ns.use_resources:
                     ns.use_resources.append(r)
+    if not ns.use_random_seed:
+        if ns.random_seed is not None:
+            parser.error("--no-use-randseed and --randseed=... don't go together")
+        if ns.randomize:
+            parser.error("--no-use-randseed and --randomize don't go together")
     if ns.random_seed is not None:
         ns.randomize = True
     if ns.verbose:

Lib/test/libregrtest/main.py

@@ -107,6 +107,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
         self.forever: bool = ns.forever
         self.randomize: bool = ns.randomize
         self.random_seed: int | None = ns.random_seed
+        self.use_random_seed: bool = ns.use_random_seed
         self.output_on_failure: bool = ns.verbose3
         self.timeout: float | None = ns.timeout
         if ns.huntrleaks:
@@ -208,11 +209,12 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList
                 print(f"Cannot find starting test: {self.starting_test}")
                 sys.exit(1)
 
-        if self.randomize:
+        if self.use_random_seed:
             if self.random_seed is None:
                 self.random_seed = random.randrange(100_000_000)
             random.seed(self.random_seed)
-            random.shuffle(selected)
+            if self.randomize:
+                random.shuffle(selected)
 
         return (tuple(selected), tests)
 
@@ -433,7 +435,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                    or tests or self.cmdline_args)):
             display_header(self.use_resources)
 
-        if self.randomize:
+        if self.use_random_seed:
             print("Using random seed", self.random_seed)
 
         runtests = self.create_run_tests(selected)

Lib/test/test_regrtest.py

@@ -949,6 +949,22 @@ def test_random(self):
         test_random2 = int(match.group(1))
         self.assertEqual(test_random2, test_random)
 
+        # check that random.seed is used by default
+        output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN)
+        self.assertIsInstance(self.parse_random_seed(output), int)
+
+        # check that --no-use-randseed disables seed
+        output = self.run_tests('--no-use-randseed', test,
+                                exitcode=EXITCODE_NO_TESTS_RAN)
+        with self.assertRaises(AssertionError):
+            self.parse_random_seed(output)
+
+        # check that --no-use-randseed and --randseed are not compatible
+        self.run_tests('--no-use-randseed', f'--randseed={randseed}', test,
+                       exitcode=EXITCODE_BAD_TEST)
+        self.run_tests('--no-use-randseed', '-r', test,
+                       exitcode=EXITCODE_BAD_TEST)
+
     def test_fromfile(self):
         # test --fromfile
         tests = [self.create_test() for index in range(5)]

Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst

@@ -0,0 +1,2 @@
+``libregrtest`` now always sets ``random.seed`` unless ``--no-use-randseed``
+is provided.