diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 7a8d38685b984c..5254866fbcf5ec 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -482,6 +482,8 @@ The :mod:`test.support` module defines the following functions: ``True`` if called by a function whose ``__name__`` is ``'__main__'``. Used when tests are executed by :mod:`test.regrtest`. + If called at the top level, sets label "requires\_\ *resource*" on the module. + .. function:: sortdict(dict) @@ -498,6 +500,18 @@ The :mod:`test.support` module defines the following functions: rather than looking directly in the path directories. +.. function:: mark(label, *, globals=None) + + Add a label to tests. + The ``@mark('label')`` decorator adds a label to method or class. + ``test.support.mark('label', globals=globals())`` adds a label to the whole + module. + + Many :mod:`test.support` decorators like :func:`requires_resource`, + :func:`~test.support.cpython_only` or :func:`bigmemtest` add labels + automatically. + + .. function:: get_pagesize() Get size of a page in bytes. @@ -736,26 +750,31 @@ The :mod:`test.support` module defines the following functions: .. decorator:: requires_zlib Decorator for skipping tests if :mod:`zlib` doesn't exist. + Adds label "requires_zlib". .. decorator:: requires_gzip Decorator for skipping tests if :mod:`gzip` doesn't exist. + Adds label "requires_gzip". .. decorator:: requires_bz2 Decorator for skipping tests if :mod:`bz2` doesn't exist. + Adds label "requires_bz2". .. decorator:: requires_lzma Decorator for skipping tests if :mod:`lzma` doesn't exist. + Adds label "requires_lzma". .. decorator:: requires_resource(resource) Decorator for skipping tests if *resource* is not available. + Adds label "requires\_\ *resource*". .. decorator:: requires_docstrings @@ -772,13 +791,16 @@ The :mod:`test.support` module defines the following functions: .. decorator:: cpython_only Decorator for tests only applicable to CPython. + Adds label "impl_detail_cpython". .. decorator:: impl_detail(msg=None, **guards) Decorator for invoking :func:`check_impl_detail` on *guards*. If that returns ``False``, then uses *msg* as the reason for skipping the test. - + For every keyword argument *name* adds a label + "impl_detail\_\ *name*" if its value is true or + "impl_detail_no\_\ *name*" otherwise. .. decorator:: no_tracing @@ -807,10 +829,13 @@ The :mod:`test.support` module defines the following functions: method may be less than the requested value. If *dry_run* is ``False``, it means the test doesn't support dummy runs when ``-M`` is not specified. + Adds label "bigmemtest". + .. decorator:: bigaddrspacetest Decorator for tests that fill the address space. + Adds label "bigaddrspacetest". .. function:: check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None) @@ -1592,6 +1617,8 @@ The :mod:`test.support.import_helper` module provides support for import tests. optional for others, set *required_on* to an iterable of platform prefixes which will be compared against :data:`sys.platform`. + If called at the top level, sets label "requires\_\ *name*" on the module. + .. versionadded:: 3.1 diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 87b926db0686ce..5cfca15eb1a6cd 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -162,6 +162,7 @@ def __init__(self, **kwargs) -> None: self.header = False self.failfast = False self.match_tests = [] + self.match_labels = [] self.pgo = False self.pgo_extended = False self.worker_json = None @@ -270,6 +271,12 @@ def _create_parser(): group.add_argument('-i', '--ignore', metavar='PAT', dest='match_tests', action=FilterAction, const=False, help='ignore test cases and methods with glob pattern PAT') + group.add_argument('--label', metavar='NAME', + dest='match_labels', action=FilterAction, const=True, + help='match test cases and methods with label NAME') + group.add_argument('--no-label', metavar='NAME', + dest='match_labels', action=FilterAction, const=False, + help='ignore test cases and methods with label NAME') group.add_argument('--matchfile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=True, diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py index 817624d79e9263..29caa7a644eeb4 100644 --- a/Lib/test/libregrtest/filter.py +++ b/Lib/test/libregrtest/filter.py @@ -1,21 +1,50 @@ import itertools import operator import re +import sys # By default, don't filter tests _test_matchers = () _test_patterns = () +_match_labels = () def match_test(test): # Function used by support.run_unittest() and regrtest --list-cases + return match_test_id(test) and match_test_label(test) + +def match_test_id(test): result = False for matcher, result in reversed(_test_matchers): if matcher(test.id()): return result return not result +def match_test_label(test): + result = False + for label, result in reversed(_match_labels): + if _has_label(test, label): + return result + return not result + +def _has_label(test, label): + attrname = f'_label_{label}' + if hasattr(test, attrname): + return True + testMethod = getattr(test, test._testMethodName) + while testMethod is not None: + if hasattr(testMethod, attrname): + return True + testMethod = getattr(testMethod, '__wrapped__', None) + try: + module = sys.modules[test.__class__.__module__] + if hasattr(module, attrname): + return True + except KeyError: + pass + return False + def _is_full_match_test(pattern): # If a pattern contains at least one dot, it's considered @@ -27,7 +56,7 @@ def _is_full_match_test(pattern): return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) -def set_match_tests(patterns): +def set_match_tests(patterns=None, match_labels=None): global _test_matchers, _test_patterns if not patterns: diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 78343775bc5b99..937f8ee119edea 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -85,9 +85,10 @@ def _list_cases(suite): def list_cases(tests: TestTuple, *, match_tests: TestFilter | None = None, + match_labels: TestFilter | None = None, test_dir: StrPath | None = None): support.verbose = False - set_match_tests(match_tests) + set_match_tests(match_tests, match_labels) skipped = [] for test_name in tests: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9b86548c89fb2e..82bda560e7a592 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -79,6 +79,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Select tests self.match_tests: TestFilter = ns.match_tests + self.match_labels: TestFilter = ns.match_labels self.exclude: bool = ns.exclude self.fromfile: StrPath | None = ns.fromfile self.starting_test: TestName | None = ns.start @@ -400,6 +401,7 @@ def create_run_tests(self, tests: TestTuple): fail_fast=self.fail_fast, fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, + match_labels=self.match_labels, match_tests_dict=None, rerun=False, forever=self.forever, @@ -652,6 +654,7 @@ def main(self, tests: TestList | None = None): elif self.want_list_cases: list_cases(selected, match_tests=self.match_tests, + match_labels=self.match_labels, test_dir=self.test_dir) else: exitcode = self.run_tests(selected, tests) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index bfed1b4a2a5817..a60681c5647e40 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -73,6 +73,7 @@ class RunTests: fail_fast: bool fail_env_changed: bool match_tests: TestFilter + match_labels: TestFilter match_tests_dict: FilterDict | None rerun: bool forever: bool diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 97edba9f87d7f9..2c4ca4d293ad21 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -93,7 +93,7 @@ def setup_tests(runtests: RunTests): support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended - set_match_tests(runtests.match_tests) + set_match_tests(runtests.match_tests, runtests.match_labels) if runtests.use_junit: support.junit_xml_list = [] diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 90fb1e670dfb38..61311b42488450 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -298,6 +298,9 @@ def is_resource_enabled(resource): def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available.""" + f = sys._getframe(1) + if f.f_globals is f.f_locals: + mark(f'requires_{resource}', globals=f.f_globals) if not is_resource_enabled(resource): if msg is None: msg = "Use of the %r resource not enabled" % resource @@ -476,28 +479,28 @@ def requires_zlib(reason='requires zlib'): import zlib except ImportError: zlib = None - return unittest.skipUnless(zlib, reason) + return skipUnless(zlib, reason, label='requires_zlib') def requires_gzip(reason='requires gzip'): try: import gzip except ImportError: gzip = None - return unittest.skipUnless(gzip, reason) + return skipUnless(gzip, reason, label='requires_gzip') def requires_bz2(reason='requires bz2'): try: import bz2 except ImportError: bz2 = None - return unittest.skipUnless(bz2, reason) + return skipUnless(bz2, reason, label='requires_bz2') def requires_lzma(reason='requires lzma'): try: import lzma except ImportError: lzma = None - return unittest.skipUnless(lzma, reason) + return skipUnless(lzma, reason, label='requires_lzma') def has_no_debug_ranges(): try: @@ -508,7 +511,7 @@ def has_no_debug_ranges(): return not bool(config['code_debug_ranges']) def requires_debug_ranges(reason='requires co_positions / debug_ranges'): - return unittest.skipIf(has_no_debug_ranges(), reason) + return skipIf(has_no_debug_ranges(), reason, label='requires_debug_ranges') MS_WINDOWS = (sys.platform == 'win32') @@ -530,28 +533,34 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi def requires_fork(): - return unittest.skipUnless(has_fork_support, "requires working os.fork()") + return skipUnless(has_fork_support, "requires working os.fork()", + label='requires_fork') has_subprocess_support = not is_emscripten and not is_wasi def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" - return unittest.skipUnless(has_subprocess_support, "requires subprocess support") + return skipUnless(has_subprocess_support, "requires subprocess support", + label='requires_subprocess') # Emscripten's socket emulation and WASI sockets have limitations. has_socket_support = not is_emscripten and not is_wasi -def requires_working_socket(*, module=False): +def requires_working_socket(*, module=False, globals=None): """Skip tests or modules that require working sockets Can be used as a function/class decorator or to skip an entire module. """ + label = 'requires_socket' msg = "requires socket support" - if module: + if module or globals is not None: + if globals is None: + globals = sys._getframe(1).f_globals + mark(label, globals=globals) if not has_socket_support: raise unittest.SkipTest(msg) else: - return unittest.skipUnless(has_socket_support, msg) + return skipUnless(has_socket_support, msg, label=label) # Does strftime() support glibc extension like '%4Y'? has_strftime_extensions = False @@ -974,6 +983,8 @@ def bigmemtest(size, memuse, dry_run=True): test doesn't support dummy runs when -M is not specified. """ def decorator(f): + @mark('bigmemtest') + @functools.wraps(f) def wrapper(self): size = wrapper.size memuse = wrapper.memuse @@ -1010,6 +1021,8 @@ def wrapper(self): def bigaddrspacetest(f): """Decorator for tests that fill the address space.""" + @mark('bigaddrspacetest') + @functools.wraps(f) def wrapper(self): if max_memuse < MAX_Py_ssize_t: if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31: @@ -1026,16 +1039,41 @@ def wrapper(self): #======================================================================= # unittest integration. -def _id(obj): - return obj +def mark(label, *, globals=None): + """Add a label to test. + + To add a label to method or class, use it as a decorator. + + To add a label to module, pass the globals() dict as the globals argument. + """ + if globals is not None: + globals[f'_label_{label}'] = True + return + def decorator(test): + setattr(test, f'_label_{label}', True) + return test + return decorator + +def combine(*decorators): + def decorator(test): + for deco in reversed(decorators): + test = deco(test) + return test + return decorator + +def skipUnless(condition, reason, *, label): + return combine(unittest.skipUnless(condition, reason), mark(label)) + +def skipIf(condition, reason, *, label): + return combine(unittest.skipIf(condition, reason), mark(label)) def requires_resource(resource): + label = 'requires_' + resource if resource == 'gui' and not _is_gui_available(): - return unittest.skip(_is_gui_available.reason) - if is_resource_enabled(resource): - return _id - else: - return unittest.skip("resource {0!r} is not enabled".format(resource)) + return skipUnless(False, _is_gui_available.reason, label=label) + return skipUnless(is_resource_enabled(resource), + f"resource {resource!r} is not enabled", + label=label) def cpython_only(test): """ @@ -1044,8 +1082,16 @@ def cpython_only(test): return impl_detail(cpython=True)(test) def impl_detail(msg=None, **guards): + guards, _ = _parse_guards(guards) + decorators = [] + for name in reversed(guards): + if guards[name]: + label = f'impl_detail_{name}' + else: + label = f'impl_detail_no_{name}' + decorators.append(mark(label)) if check_impl_detail(**guards): - return _id + return combine(*decorators) if msg is None: guardnames, default = _parse_guards(guards) if default: @@ -1054,7 +1100,7 @@ def impl_detail(msg=None, **guards): msg = "implementation detail specific to {0}" guardnames = sorted(guardnames.keys()) msg = msg.format(' or '.join(guardnames)) - return unittest.skip(msg) + return combine(unittest.skip(msg), *decorators) def _parse_guards(guards): # Returns a tuple ({platform_name: run_me}, default_value) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 3d804f2b590108..b540e4dc90d2ad 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -74,6 +74,10 @@ def import_module(name, deprecated=False, *, required_on=()): compared against sys.platform. """ with _ignore_deprecated_imports(deprecated): + f = sys._getframe(1) + if f.f_globals is f.f_locals: + from test.support import mark + mark(f'requires_{name}', globals=f.f_globals) try: return importlib.import_module(name) except ImportError as msg: diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 87941ee1791b4e..229859bdb9beb7 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -147,6 +147,7 @@ def _is_ipv6_enabled(): _bind_nix_socket_error = None def skip_unless_bind_unix_socket(test): """Decorator for tests requiring a functional bind() for unix sockets.""" + test = support.mark('requires_unix_sockets')(test) if not hasattr(socket, 'AF_UNIX'): return unittest.skip('No UNIX Sockets')(test) global _bind_nix_socket_error diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index 7f16050f32b9d1..67dcb0cc6e8eb3 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -234,14 +234,18 @@ def _can_start_thread() -> bool: can_start_thread = _can_start_thread() -def requires_working_threading(*, module=False): +def requires_working_threading(*, module=False, globals=None): """Skip tests or modules that require working threading. Can be used as a function/class decorator or to skip an entire module. """ + label = 'requires_threading' msg = "requires threading support" - if module: + if module or globals is not None: + if globals is None: + globals = sys._getframe(1).f_globals + support.mark(label, globals=globals) if not can_start_thread: raise unittest.SkipTest(msg) else: - return unittest.skipUnless(can_start_thread, msg) + return support.skipUnless(can_start_thread, msg, label=label) diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-12-53-53.gh-issue-108828.zoWIyX.rst b/Misc/NEWS.d/next/Tests/2023-09-03-12-53-53.gh-issue-108828.zoWIyX.rst new file mode 100644 index 00000000000000..cad3adc7290633 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-12-53-53.gh-issue-108828.zoWIyX.rst @@ -0,0 +1,7 @@ +Add support of labels in tests. The ``@test.support.mark('label')`` +decorator adds a label to method or class. ``test.support.mark('label', +globals=globals())`` adds a label to the whole module. Many +:mod:`test.support` decorators like :func:`~test.support.requires_resource`, +:func:`~test.support.cpython_only` or :func:`~test.support.bigmemtest` add +labels automatically. Tests which have or have not the specified label can +be filtered by options ``--label`` and ``--no-label``.