From 56deae680f05c890a1cee78aa25eb03471b538e4 Mon Sep 17 00:00:00 2001 From: Lev Kats Date: Mon, 7 Jul 2025 03:21:50 +0300 Subject: [PATCH] Add repeat option to run untill tests fail Add the `--repeat` command line option so each test should succeed `--repeat` times to pass. One may think of the option as a `--retries` analog, but instead of assuring that each test passes at least once, we require each test to pass every time. Combining both non-trivial `--retries` and `--repeat` options values require each test to pass at least once in `--retries` runs `--repeat` times in a row. Needed for tarantool/tarantool#6646 --- lib/options.py | 16 ++++++++++++++++ lib/worker.py | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/options.py b/lib/options.py index 12021b35..5a39d107 100644 --- a/lib/options.py +++ b/lib/options.py @@ -319,6 +319,22 @@ def __init__(self): Default: ${TEST_RUN_RETRIES} or 0. """)) + parser.add_argument( + "--repeat", + dest='repeat', + default=env_int('TEST_RUN_REPEAT', 1), + type=int, + help=format_help( + """ + The number of times each test will be repeated. + + Combining both non-trivial --retries and --repeat options + values require each test to pass at least once in + --retries runs --repeat times in a row. + + Default: ${TEST_RUN_REPEAT} or 1. + """)) + parser.add_argument( "-p", "--pattern", dest='pattern', diff --git a/lib/worker.py b/lib/worker.py index bf5d199d..40e9cb44 100644 --- a/lib/worker.py +++ b/lib/worker.py @@ -96,14 +96,15 @@ def get_task_groups(): """ suites = find_suites() res = collections.OrderedDict() + repeat = Options().args.repeat for suite in suites: key = os.path.basename(suite.suite_path) gen_worker = functools.partial(Worker, suite) # get _id as an arg # Split stable and fragile tasks to two groups. Run stable # tasks in parallel with other ones. Run fragile tasks one # by one when all parallel tasks will be finished. - stable_task_ids = [task.id for task in suite.stable_tests()] - fragile_task_ids = [task.id for task in suite.fragile_tests()] + stable_task_ids = [task.id for task in suite.stable_tests()] * repeat + fragile_task_ids = [task.id for task in suite.fragile_tests()] * repeat if stable_task_ids: res[key] = { 'gen_worker': gen_worker,