diff options
Diffstat (limited to 'Lib/test')
73 files changed, 3415 insertions, 1732 deletions
diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index 274beb65a6d..70c44da2ec6 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -57,6 +57,13 @@ def spam_with_globals_and_builtins(): print(res) +def spam_with_global_and_attr_same_name(): + try: + spam_minimal.spam_minimal + except AttributeError: + pass + + def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs): return (a, b, c, d, e, f, args, kwargs) @@ -190,6 +197,7 @@ TOP_FUNCTIONS = [ spam_minimal, spam_with_builtins, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, spam_full_args, spam_full_args_with_defaults, spam_args_attrs_and_builtins, @@ -258,6 +266,7 @@ STATELESS_CODE = [ script_with_globals, spam_full_args_with_defaults, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, spam_full, ] @@ -275,6 +284,7 @@ SCRIPT_FUNCTIONS = [ *PURE_SCRIPT_FUNCTIONS, script_with_globals, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, ] diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 75f31d858d3..a1259ff1d63 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6801,6 +6801,35 @@ class _TestSpawnedSysPath(BaseTestCase): self.assertEqual(child_sys_path[1:], sys.path[1:]) self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") + def test_std_streams_flushed_after_preload(self): + # gh-135335: Check fork server flushes standard streams after + # preloading modules + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + # Create a test module in the temporary directory on the child's path + # TODO: This can all be simplified once gh-126631 is fixed and we can + # use __main__ instead of a module. + dirname = os.path.join(self._temp_dir, 'preloaded_module') + init_name = os.path.join(dirname, '__init__.py') + os.mkdir(dirname) + with open(init_name, "w") as f: + cmd = '''if 1: + import sys + print('stderr', end='', file=sys.stderr) + print('stdout', end='', file=sys.stdout) + ''' + f.write(cmd) + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_flush.py') + env = {'PYTHONPATH': self._temp_dir} + _, out, err = test.support.script_helper.assert_python_ok(name, **env) + + # Check stderr first, as it is more likely to be useful to see in the + # event of a failure. + self.assertEqual(err.decode().rstrip(), 'stderr') + self.assertEqual(out.decode().rstrip(), 'stdout') + class MiscTestCase(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 08b638e4b8d..6884ac0dbe6 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -643,6 +643,34 @@ def test_assert_unicode(): else: raise RuntimeError("Expected sys.audit(9) to fail.") +def test_sys_remote_exec(): + import tempfile + + pid = os.getpid() + event_pid = -1 + event_script_path = "" + remote_event_script_path = "" + def hook(event, args): + if event not in ["sys.remote_exec", "cpython.remote_debugger_script"]: + return + print(event, args) + match event: + case "sys.remote_exec": + nonlocal event_pid, event_script_path + event_pid = args[0] + event_script_path = args[1] + case "cpython.remote_debugger_script": + nonlocal remote_event_script_path + remote_event_script_path = args[0] + + sys.addaudithook(hook) + with tempfile.NamedTemporaryFile(mode='w+', delete=True) as tmp_file: + tmp_file.write("a = 1+1\n") + tmp_file.flush() + sys.remote_exec(pid, tmp_file.name) + assertEqual(event_pid, pid) + assertEqual(event_script_path, tmp_file.name) + assertEqual(remote_event_script_path, tmp_file.name) if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 0d9c059a938..a2d01b157ac 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -190,6 +190,12 @@ class Regrtest: strip_py_suffix(tests) + exclude_tests = set() + if self.exclude: + for arg in self.cmdline_args: + exclude_tests.add(arg) + self.cmdline_args = [] + if self.pgo: # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) @@ -200,17 +206,15 @@ class Regrtest: if self.tsan_parallel: setup_tsan_parallel_tests(self.cmdline_args) - exclude_tests = set() - if self.exclude: - for arg in self.cmdline_args: - exclude_tests.add(arg) - self.cmdline_args = [] - alltests = findtests(testdir=self.test_dir, exclude=exclude_tests) if not self.fromfile: selected = tests or self.cmdline_args + if exclude_tests: + # Support "--pgo/--tsan -x test_xxx" command + selected = [name for name in selected + if name not in exclude_tests] if selected: selected = split_test_packages(selected) else: diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 57d7b649d2e..958a915626a 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -283,7 +283,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: try: setup_tests(runtests) - if output_on_failure: + if output_on_failure or runtests.pgo: support.verbose = True stream = io.StringIO() diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index d984a735bdf..3545c5f999f 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -8,7 +8,7 @@ TSAN_TESTS = [ 'test_capi.test_pyatomic', 'test_code', 'test_ctypes', - # 'test_concurrent_futures', # gh-130605: too many data races + 'test_concurrent_futures', 'test_enum', 'test_functools', 'test_httpservers', diff --git a/Lib/test/mp_preload_flush.py b/Lib/test/mp_preload_flush.py new file mode 100644 index 00000000000..3501554d366 --- /dev/null +++ b/Lib/test/mp_preload_flush.py @@ -0,0 +1,15 @@ +import multiprocessing +import sys + +modname = 'preloaded_module' +if __name__ == '__main__': + if modname in sys.modules: + raise AssertionError(f'{modname!r} is not in sys.modules') + multiprocessing.set_start_method('forkserver') + multiprocessing.set_forkserver_preload([modname]) + for _ in range(2): + p = multiprocessing.Process() + p.start() + p.join() +elif modname not in sys.modules: + raise AssertionError(f'{modname!r} is not in sys.modules') diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 9d6ae3e4d00..9a3a26a8400 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1100,6 +1100,11 @@ class AbstractUnpickleTests: self.check_unpickling_error((pickle.UnpicklingError, OverflowError), dumped) + def test_large_binstring(self): + errmsg = 'BINSTRING pickle has negative byte count' + with self.assertRaisesRegex(pickle.UnpicklingError, errmsg): + self.loads(b'T\0\0\0\x80') + def test_get(self): pickled = b'((lp100000\ng100000\nt.' unpickled = self.loads(pickled) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 48e74adcce3..fd39d3f7c95 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -46,6 +46,7 @@ __all__ = [ # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", + "support_remote_exec_only", # os "get_pagesize", # network @@ -2332,6 +2333,7 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): qualname = f"{name}" msg = f"cannot create '{re.escape(qualname)}' instances" testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) + testcase.assertRaisesRegex(TypeError, msg, tp.__new__, tp, *args, **kwds) def get_recursion_depth(): """Get the recursion depth of the caller function. @@ -3069,6 +3071,27 @@ def is_libssl_fips_mode(): return False # more of a maybe, unless we add this to the _ssl module. return get_fips_mode() != 0 +def _supports_remote_attaching(): + PROCESS_VM_READV_SUPPORTED = False + + try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + except ImportError: + pass + + return PROCESS_VM_READV_SUPPORTED + +def _support_remote_exec_only_impl(): + if not sys.is_remote_debug_enabled(): + return unittest.skip("Remote debugging is not enabled") + if sys.platform not in ("darwin", "linux", "win32"): + return unittest.skip("Test only runs on Linux, Windows and macOS") + if sys.platform == "linux" and not _supports_remote_attaching(): + return unittest.skip("Test only runs on Linux with process_vm_readv support") + return _id + +def support_remote_exec_only(test): + return _support_remote_exec_only_impl()(test) class EqualToForwardRef: """Helper to ease use of annotationlib.ForwardRef in tests. diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/channels.py index 1724759b75a..b2de24d9d3e 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/channels.py @@ -2,14 +2,14 @@ import time import _interpchannels as _channels -from . import _crossinterp +from concurrent.interpreters import _crossinterp # aliases: from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, # noqa: F401 ChannelEmptyError, ChannelNotEmptyError, # noqa: F401 ) -from ._crossinterp import ( +from concurrent.interpreters._crossinterp import ( UNBOUND_ERROR, UNBOUND_REMOVE, ) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py deleted file mode 100644 index 6d1b0690805..00000000000 --- a/Lib/test/support/interpreters/__init__.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Subinterpreters High Level Module.""" - -import threading -import weakref -import _interpreters - -# aliases: -from _interpreters import ( - InterpreterError, InterpreterNotFoundError, NotShareableError, - is_shareable, -) - - -__all__ = [ - 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', - 'Interpreter', - 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', - 'NotShareableError', - 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', -] - - -_queuemod = None - -def __getattr__(name): - if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): - global create_queue, Queue, QueueEmpty, QueueFull - ns = globals() - from .queues import ( - create as create_queue, - Queue, QueueEmpty, QueueFull, - ) - return ns[name] - else: - raise AttributeError(name) - - -_EXEC_FAILURE_STR = """ -{superstr} - -Uncaught in the interpreter: - -{formatted} -""".strip() - -class ExecutionFailed(InterpreterError): - """An unhandled exception happened during execution. - - This is raised from Interpreter.exec() and Interpreter.call(). - """ - - def __init__(self, excinfo): - msg = excinfo.formatted - if not msg: - if excinfo.type and excinfo.msg: - msg = f'{excinfo.type.__name__}: {excinfo.msg}' - else: - msg = excinfo.type.__name__ or excinfo.msg - super().__init__(msg) - self.excinfo = excinfo - - def __str__(self): - try: - formatted = self.excinfo.errdisplay - except Exception: - return super().__str__() - else: - return _EXEC_FAILURE_STR.format( - superstr=super().__str__(), - formatted=formatted, - ) - - -def create(): - """Return a new (idle) Python interpreter.""" - id = _interpreters.create(reqrefs=True) - return Interpreter(id, _ownsref=True) - - -def list_all(): - """Return all existing interpreters.""" - return [Interpreter(id, _whence=whence) - for id, whence in _interpreters.list_all(require_ready=True)] - - -def get_current(): - """Return the currently running interpreter.""" - id, whence = _interpreters.get_current() - return Interpreter(id, _whence=whence) - - -def get_main(): - """Return the main interpreter.""" - id, whence = _interpreters.get_main() - assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) - return Interpreter(id, _whence=whence) - - -_known = weakref.WeakValueDictionary() - -class Interpreter: - """A single Python interpreter. - - Attributes: - - "id" - the unique process-global ID number for the interpreter - "whence" - indicates where the interpreter was created - - If the interpreter wasn't created by this module - then any method that modifies the interpreter will fail, - i.e. .close(), .prepare_main(), .exec(), and .call() - """ - - _WHENCE_TO_STR = { - _interpreters.WHENCE_UNKNOWN: 'unknown', - _interpreters.WHENCE_RUNTIME: 'runtime init', - _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', - _interpreters.WHENCE_CAPI: 'C-API', - _interpreters.WHENCE_XI: 'cross-interpreter C-API', - _interpreters.WHENCE_STDLIB: '_interpreters module', - } - - def __new__(cls, id, /, _whence=None, _ownsref=None): - # There is only one instance for any given ID. - if not isinstance(id, int): - raise TypeError(f'id must be an int, got {id!r}') - id = int(id) - if _whence is None: - if _ownsref: - _whence = _interpreters.WHENCE_STDLIB - else: - _whence = _interpreters.whence(id) - assert _whence in cls._WHENCE_TO_STR, repr(_whence) - if _ownsref is None: - _ownsref = (_whence == _interpreters.WHENCE_STDLIB) - try: - self = _known[id] - assert hasattr(self, '_ownsref') - except KeyError: - self = super().__new__(cls) - _known[id] = self - self._id = id - self._whence = _whence - self._ownsref = _ownsref - if _ownsref: - # This may raise InterpreterNotFoundError: - _interpreters.incref(id) - return self - - def __repr__(self): - return f'{type(self).__name__}({self.id})' - - def __hash__(self): - return hash(self._id) - - def __del__(self): - self._decref() - - # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None - - def _decref(self): - if not self._ownsref: - return - self._ownsref = False - try: - _interpreters.decref(self._id) - except InterpreterNotFoundError: - pass - - @property - def id(self): - return self._id - - @property - def whence(self): - return self._WHENCE_TO_STR[self._whence] - - def is_running(self): - """Return whether or not the identified interpreter is running.""" - return _interpreters.is_running(self._id) - - # Everything past here is available only to interpreters created by - # interpreters.create(). - - def close(self): - """Finalize and destroy the interpreter. - - Attempting to destroy the current interpreter results - in an InterpreterError. - """ - return _interpreters.destroy(self._id, restrict=True) - - def prepare_main(self, ns=None, /, **kwargs): - """Bind the given values into the interpreter's __main__. - - The values must be shareable. - """ - ns = dict(ns, **kwargs) if ns is not None else kwargs - _interpreters.set___main___attrs(self._id, ns, restrict=True) - - def exec(self, code, /): - """Run the given source code in the interpreter. - - This is essentially the same as calling the builtin "exec" - with this interpreter, using the __dict__ of its __main__ - module as both globals and locals. - - There is no return value. - - If the code raises an unhandled exception then an ExecutionFailed - exception is raised, which summarizes the unhandled exception. - The actual exception is discarded because objects cannot be - shared between interpreters. - - This blocks the current Python thread until done. During - that time, the previous interpreter is allowed to run - in other threads. - """ - excinfo = _interpreters.exec(self._id, code, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) - - def _call(self, callable, args, kwargs): - res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) - return res - - def call(self, callable, /, *args, **kwargs): - """Call the object in the interpreter with given args/kwargs. - - Nearly all callables, args, kwargs, and return values are - supported. All "shareable" objects are supported, as are - "stateless" functions (meaning non-closures that do not use - any globals). This method will fall back to pickle. - - If the callable raises an exception then the error display - (including full traceback) is sent back between the interpreters - and an ExecutionFailed exception is raised, much like what - happens with Interpreter.exec(). - """ - return self._call(callable, args, kwargs) - - def call_in_thread(self, callable, /, *args, **kwargs): - """Return a new thread that calls the object in the interpreter. - - The return value and any raised exception are discarded. - """ - t = threading.Thread(target=self._call, args=(callable, args, kwargs)) - t.start() - return t diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/test/support/interpreters/_crossinterp.py deleted file mode 100644 index 544e197ba4c..00000000000 --- a/Lib/test/support/interpreters/_crossinterp.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Common code between queues and channels.""" - - -class ItemInterpreterDestroyed(Exception): - """Raised when trying to get an item whose interpreter was destroyed.""" - - -class classonly: - """A non-data descriptor that makes a value only visible on the class. - - This is like the "classmethod" builtin, but does not show up on - instances of the class. It may be used as a decorator. - """ - - def __init__(self, value): - self.value = value - self.getter = classmethod(value).__get__ - self.name = None - - def __set_name__(self, cls, name): - if self.name is not None: - raise TypeError('already used') - self.name = name - - def __get__(self, obj, cls): - if obj is not None: - raise AttributeError(self.name) - # called on the class - return self.getter(None, cls) - - -class UnboundItem: - """Represents a cross-interpreter item no longer bound to an interpreter. - - An item is unbound when the interpreter that added it to the - cross-interpreter container is destroyed. - """ - - __slots__ = () - - @classonly - def singleton(cls, kind, module, name='UNBOUND'): - doc = cls.__doc__.replace('cross-interpreter container', kind) - doc = doc.replace('cross-interpreter', kind) - subclass = type( - f'Unbound{kind.capitalize()}Item', - (cls,), - dict( - _MODULE=module, - _NAME=name, - __doc__=doc, - ), - ) - return object.__new__(subclass) - - _MODULE = __name__ - _NAME = 'UNBOUND' - - def __new__(cls): - raise Exception(f'use {cls._MODULE}.{cls._NAME}') - - def __repr__(self): - return f'{self._MODULE}.{self._NAME}' -# return f'interpreters.queues.UNBOUND' - - -UNBOUND = object.__new__(UnboundItem) -UNBOUND_ERROR = object() -UNBOUND_REMOVE = object() - -_UNBOUND_CONSTANT_TO_FLAG = { - UNBOUND_REMOVE: 1, - UNBOUND_ERROR: 2, - UNBOUND: 3, -} -_UNBOUND_FLAG_TO_CONSTANT = {v: k - for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()} - - -def serialize_unbound(unbound): - op = unbound - try: - flag = _UNBOUND_CONSTANT_TO_FLAG[op] - except KeyError: - raise NotImplementedError(f'unsupported unbound replacement op {op!r}') - return flag, - - -def resolve_unbound(flag, exctype_destroyed): - try: - op = _UNBOUND_FLAG_TO_CONSTANT[flag] - except KeyError: - raise NotImplementedError(f'unsupported unbound replacement op {flag!r}') - if op is UNBOUND_REMOVE: - # "remove" not possible here - raise NotImplementedError - elif op is UNBOUND_ERROR: - raise exctype_destroyed("item's original interpreter destroyed") - elif op is UNBOUND: - return UNBOUND - else: - raise NotImplementedError(repr(op)) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py deleted file mode 100644 index 99987f2f692..00000000000 --- a/Lib/test/support/interpreters/queues.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Cross-interpreter Queues High Level Module.""" - -import queue -import time -import weakref -import _interpqueues as _queues -from . import _crossinterp - -# aliases: -from _interpqueues import ( - QueueError, QueueNotFoundError, -) -from ._crossinterp import ( - UNBOUND_ERROR, UNBOUND_REMOVE, -) - -__all__ = [ - 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', - 'create', 'list_all', - 'Queue', - 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', - 'ItemInterpreterDestroyed', -] - - -class QueueEmpty(QueueError, queue.Empty): - """Raised from get_nowait() when the queue is empty. - - It is also raised from get() if it times out. - """ - - -class QueueFull(QueueError, queue.Full): - """Raised from put_nowait() when the queue is full. - - It is also raised from put() if it times out. - """ - - -class ItemInterpreterDestroyed(QueueError, - _crossinterp.ItemInterpreterDestroyed): - """Raised from get() and get_nowait().""" - - -_SHARED_ONLY = 0 -_PICKLED = 1 - - -UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) - - -def _serialize_unbound(unbound): - if unbound is UNBOUND: - unbound = _crossinterp.UNBOUND - return _crossinterp.serialize_unbound(unbound) - - -def _resolve_unbound(flag): - resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) - if resolved is _crossinterp.UNBOUND: - resolved = UNBOUND - return resolved - - -def create(maxsize=0, *, unbounditems=UNBOUND): - """Return a new cross-interpreter queue. - - The queue may be used to pass data safely between interpreters. - - "unbounditems" sets the default for Queue.put(); see that method for - supported values. The default value is UNBOUND, which replaces - the unbound item. - """ - unbound = _serialize_unbound(unbounditems) - unboundop, = unbound - qid = _queues.create(maxsize, unboundop, -1) - self = Queue(qid) - self._set_unbound(unboundop, unbounditems) - return self - - -def list_all(): - """Return a list of all open queues.""" - queues = [] - for qid, unboundop, _ in _queues.list_all(): - self = Queue(qid) - if not hasattr(self, '_unbound'): - self._set_unbound(unboundop) - else: - assert self._unbound[0] == unboundop - queues.append(self) - return queues - - -_known_queues = weakref.WeakValueDictionary() - -class Queue: - """A cross-interpreter queue.""" - - def __new__(cls, id, /): - # There is only one instance for any given ID. - if isinstance(id, int): - id = int(id) - else: - raise TypeError(f'id must be an int, got {id!r}') - try: - self = _known_queues[id] - except KeyError: - self = super().__new__(cls) - self._id = id - _known_queues[id] = self - _queues.bind(id) - return self - - def __del__(self): - try: - _queues.release(self._id) - except QueueNotFoundError: - pass - try: - del _known_queues[self._id] - except KeyError: - pass - - def __repr__(self): - return f'{type(self).__name__}({self.id})' - - def __hash__(self): - return hash(self._id) - - # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None - - def _set_unbound(self, op, items=None): - assert not hasattr(self, '_unbound') - if items is None: - items = _resolve_unbound(op) - unbound = (op, items) - self._unbound = unbound - return unbound - - @property - def id(self): - return self._id - - @property - def unbounditems(self): - try: - _, items = self._unbound - except AttributeError: - op, _ = _queues.get_queue_defaults(self._id) - _, items = self._set_unbound(op) - return items - - @property - def maxsize(self): - try: - return self._maxsize - except AttributeError: - self._maxsize = _queues.get_maxsize(self._id) - return self._maxsize - - def empty(self): - return self.qsize() == 0 - - def full(self): - return _queues.is_full(self._id) - - def qsize(self): - return _queues.get_count(self._id) - - def put(self, obj, timeout=None, *, - unbounditems=None, - _delay=10 / 1000, # 10 milliseconds - ): - """Add the object to the queue. - - This blocks while the queue is full. - - For most objects, the object received through Queue.get() will - be a new one, equivalent to the original and not sharing any - actual underlying data. The notable exceptions include - cross-interpreter types (like Queue) and memoryview, where the - underlying data is actually shared. Furthermore, some types - can be sent through a queue more efficiently than others. This - group includes various immutable types like int, str, bytes, and - tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable(). - - "unbounditems" controls the behavior of Queue.get() for the given - object if the current interpreter (calling put()) is later - destroyed. - - If "unbounditems" is None (the default) then it uses the - queue's default, set with create_queue(), - which is usually UNBOUND. - - If "unbounditems" is UNBOUND_ERROR then get() will raise an - ItemInterpreterDestroyed exception if the original interpreter - has been destroyed. This does not otherwise affect the queue; - the next call to put() will work like normal, returning the next - item in the queue. - - If "unbounditems" is UNBOUND_REMOVE then the item will be removed - from the queue as soon as the original interpreter is destroyed. - Be aware that this will introduce an imbalance between put() - and get() calls. - - If "unbounditems" is UNBOUND then it is returned by get() in place - of the unbound item. - """ - if unbounditems is None: - unboundop = -1 - else: - unboundop, = _serialize_unbound(unbounditems) - if timeout is not None: - timeout = int(timeout) - if timeout < 0: - raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout - while True: - try: - _queues.put(self._id, obj, unboundop) - except QueueFull as exc: - if timeout is not None and time.time() >= end: - raise # re-raise - time.sleep(_delay) - else: - break - - def put_nowait(self, obj, *, unbounditems=None): - if unbounditems is None: - unboundop = -1 - else: - unboundop, = _serialize_unbound(unbounditems) - _queues.put(self._id, obj, unboundop) - - def get(self, timeout=None, *, - _delay=10 / 1000, # 10 milliseconds - ): - """Return the next object from the queue. - - This blocks while the queue is empty. - - If the next item's original interpreter has been destroyed - then the "next object" is determined by the value of the - "unbounditems" argument to put(). - """ - if timeout is not None: - timeout = int(timeout) - if timeout < 0: - raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout - while True: - try: - obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: - if timeout is not None and time.time() >= end: - raise # re-raise - time.sleep(_delay) - else: - break - if unboundop is not None: - assert obj is None, repr(obj) - return _resolve_unbound(unboundop) - return obj - - def get_nowait(self): - """Return the next object from the channel. - - If the queue is empty then raise QueueEmpty. Otherwise this - is the same as get(). - """ - try: - obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: - raise # re-raise - if unboundop is not None: - assert obj is None, repr(obj) - return _resolve_unbound(unboundop) - return obj - - -_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index 88eee03a3de..858d31a73cf 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -9,7 +9,7 @@ import unittest from test.support import import_helper, skip_if_sanitizer _channels = import_helper.import_module('_interpchannels') -from test.support.interpreters import _crossinterp +from concurrent.interpreters import _crossinterp from test.test__interpreters import ( _interpreters, _run_output, diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index ad3ebbfdff6..a32d5d81d2b 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -485,6 +485,21 @@ class CommonTests(TestBase): msg = r'_interpreters.run_func\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_func(self.id, lambda: None, shared=1) + # See https://github.com/python/cpython/issues/135855 + msg = r'_interpreters.set___main___attrs\(\) argument 2 must be dict, not int' + with self.assertRaisesRegex(TypeError, msg): + _interpreters.set___main___attrs(self.id, 1) + + def test_invalid_shared_none(self): + msg = r'must be dict, not None' + with self.assertRaisesRegex(TypeError, msg): + _interpreters.exec(self.id, 'a', shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_string(self.id, 'a', shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_func(self.id, lambda: None, shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.set___main___attrs(self.id, None) def test_invalid_shared_encoding(self): # See https://github.com/python/cpython/issues/127196 diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index fe091e52a86..ae0e73f08c5 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1650,9 +1650,11 @@ class TestForwardRefClass(unittest.TestCase): with support.swap_attr(builtins, "int", dict): self.assertIs(ForwardRef("int").evaluate(), dict) - with self.assertRaises(NameError): + with self.assertRaises(NameError, msg="name 'doesntexist' is not defined") as exc: ForwardRef("doesntexist").evaluate() + self.assertEqual(exc.exception.name, "doesntexist") + def test_fwdref_invalid_syntax(self): fr = ForwardRef("if") with self.assertRaises(SyntaxError): diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index ba36e759ccd..34e94830204 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -2,6 +2,13 @@ import unittest from asyncio import tools +from collections import namedtuple + +FrameInfo = namedtuple('FrameInfo', ['funcname', 'filename', 'lineno']) +CoroInfo = namedtuple('CoroInfo', ['call_stack', 'task_name']) +TaskInfo = namedtuple('TaskInfo', ['task_id', 'task_name', 'coroutine_stack', 'awaited_by']) +AwaitedInfo = namedtuple('AwaitedInfo', ['thread_id', 'awaited_by']) + # mock output of get_all_awaited_by function. TEST_INPUTS_TREE = [ @@ -10,81 +17,151 @@ TEST_INPUTS_TREE = [ # different subtasks part of a TaskGroup (root1 and root2) which call # awaiter functions. ( - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "timer", - [ - [[("awaiter3", "/path/to/app.py", 130), - ("awaiter2", "/path/to/app.py", 120), - ("awaiter", "/path/to/app.py", 110)], 4], - [[("awaiterB3", "/path/to/app.py", 190), - ("awaiterB2", "/path/to/app.py", 180), - ("awaiterB", "/path/to/app.py", 170)], 5], - [[("awaiterB3", "/path/to/app.py", 190), - ("awaiterB2", "/path/to/app.py", 180), - ("awaiterB", "/path/to/app.py", 170)], 6], - [[("awaiter3", "/path/to/app.py", 130), - ("awaiter2", "/path/to/app.py", 120), - ("awaiter", "/path/to/app.py", 110)], 7], - ], - ), - ( - 8, - "root1", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 9, - "root2", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 4, - "child1_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 6, - "child2_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 7, - "child1_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], - ), - ( - 5, - "child2_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ], + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ @@ -130,26 +207,96 @@ TEST_INPUTS_TREE = [ [ # test case containing two roots ( - ( - 9, - [ - (5, "Task-5", []), - (6, "Task-6", [[["main2"], 5]]), - (7, "Task-7", [[["main2"], 5]]), - (8, "Task-8", [[["main2"], 5]]), - ], + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] ), - ( - 10, - [ - (1, "Task-1", []), - (2, "Task-2", [[["main"], 1]]), - (3, "Task-3", [[["main"], 1]]), - (4, "Task-4", [[["main"], 1]]), - ], + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] ), - (11, []), - (0, []), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ @@ -174,18 +321,63 @@ TEST_INPUTS_TREE = [ # test case containing two roots, one of them without subtasks ( [ - (1, [(2, "Task-5", [])]), - ( - 3, - [ - (4, "Task-1", []), - (5, "Task-2", [[["main"], 4]]), - (6, "Task-3", [[["main"], 4]]), - (7, "Task-4", [[["main"], 4]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] ), - (8, []), - (0, []), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( @@ -208,19 +400,44 @@ TEST_INPUTS_CYCLES_TREE = [ # this test case contains a cycle: two tasks awaiting each other. ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "a", - [[["awaiter2"], 4], [["main"], 2]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - (4, "b", [[["awaiter"], 3]]), - ], + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ([[4, 3, 4]]), @@ -229,32 +446,85 @@ TEST_INPUTS_CYCLES_TREE = [ # this test case contains two cycles ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "A", - [[["nested", "nested", "task_b"], 4]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ( - 4, - "B", - [ - [["nested", "nested", "task_c"], 5], - [["nested", "nested", "task_a"], 3], - ], + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] ), - (5, "C", [[["nested", "nested"], 6]]), - ( - 6, - "Task-2", - [[["nested", "nested", "task_b"], 4]], + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] ), - ], + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ([[4, 3, 4], [4, 6, 5, 4]]), @@ -267,81 +537,160 @@ TEST_INPUTS_TABLE = [ # different subtasks part of a TaskGroup (root1 and root2) which call # awaiter functions. ( - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "timer", - [ - [["awaiter3", "awaiter2", "awaiter"], 4], - [["awaiter1_3", "awaiter1_2", "awaiter1"], 5], - [["awaiter1_3", "awaiter1_2", "awaiter1"], 6], - [["awaiter3", "awaiter2", "awaiter"], 7], - ], - ), - ( - 8, - "root1", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 9, - "root2", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 4, - "child1_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 6, - "child2_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 7, - "child1_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], - ), - ( - 5, - "child2_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ], + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], + [1, "0x2", "Task-1", "", "", "", "0x0"], [ 1, "0x3", "timer", + "", "awaiter3 -> awaiter2 -> awaiter", "child1_1", "0x4", @@ -350,6 +699,7 @@ TEST_INPUTS_TABLE = [ 1, "0x3", "timer", + "", "awaiter1_3 -> awaiter1_2 -> awaiter1", "child2_2", "0x5", @@ -358,6 +708,7 @@ TEST_INPUTS_TABLE = [ 1, "0x3", "timer", + "", "awaiter1_3 -> awaiter1_2 -> awaiter1", "child2_1", "0x6", @@ -366,6 +717,7 @@ TEST_INPUTS_TABLE = [ 1, "0x3", "timer", + "", "awaiter3 -> awaiter2 -> awaiter", "child1_2", "0x7", @@ -374,6 +726,7 @@ TEST_INPUTS_TABLE = [ 1, "0x8", "root1", + "", "_aexit -> __aexit__ -> main", "Task-1", "0x2", @@ -382,6 +735,7 @@ TEST_INPUTS_TABLE = [ 1, "0x9", "root2", + "", "_aexit -> __aexit__ -> main", "Task-1", "0x2", @@ -390,6 +744,7 @@ TEST_INPUTS_TABLE = [ 1, "0x4", "child1_1", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root1", "0x8", @@ -398,6 +753,7 @@ TEST_INPUTS_TABLE = [ 1, "0x6", "child2_1", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root1", "0x8", @@ -406,6 +762,7 @@ TEST_INPUTS_TABLE = [ 1, "0x7", "child1_2", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root2", "0x9", @@ -414,6 +771,7 @@ TEST_INPUTS_TABLE = [ 1, "0x5", "child2_2", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root2", "0x9", @@ -424,37 +782,107 @@ TEST_INPUTS_TABLE = [ [ # test case containing two roots ( - ( - 9, - [ - (5, "Task-5", []), - (6, "Task-6", [[["main2"], 5]]), - (7, "Task-7", [[["main2"], 5]]), - (8, "Task-8", [[["main2"], 5]]), - ], + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] ), - ( - 10, - [ - (1, "Task-1", []), - (2, "Task-2", [[["main"], 1]]), - (3, "Task-3", [[["main"], 1]]), - (4, "Task-4", [[["main"], 1]]), - ], + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] ), - (11, []), - (0, []), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ - [9, "0x5", "Task-5", "", "", "0x0"], - [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], - [9, "0x7", "Task-7", "main2", "Task-5", "0x5"], - [9, "0x8", "Task-8", "main2", "Task-5", "0x5"], - [10, "0x1", "Task-1", "", "", "0x0"], - [10, "0x2", "Task-2", "main", "Task-1", "0x1"], - [10, "0x3", "Task-3", "main", "Task-1", "0x1"], - [10, "0x4", "Task-4", "main", "Task-1", "0x1"], + [9, "0x5", "Task-5", "", "", "", "0x0"], + [9, "0x6", "Task-6", "", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "", "0x0"], + [10, "0x2", "Task-2", "", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "", "main", "Task-1", "0x1"], ] ), ], @@ -462,27 +890,72 @@ TEST_INPUTS_TABLE = [ # test case containing two roots, one of them without subtasks ( [ - (1, [(2, "Task-5", [])]), - ( - 3, - [ - (4, "Task-1", []), - (5, "Task-2", [[["main"], 4]]), - (6, "Task-3", [[["main"], 4]]), - (7, "Task-4", [[["main"], 4]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] ), - (8, []), - (0, []), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-5", "", "", "0x0"], - [3, "0x4", "Task-1", "", "", "0x0"], - [3, "0x5", "Task-2", "main", "Task-1", "0x4"], - [3, "0x6", "Task-3", "main", "Task-1", "0x4"], - [3, "0x7", "Task-4", "main", "Task-1", "0x4"], + [1, "0x2", "Task-5", "", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "", "0x0"], + [3, "0x5", "Task-2", "", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "", "main", "Task-1", "0x4"], ] ), ], @@ -491,27 +964,52 @@ TEST_INPUTS_TABLE = [ # this test case contains a cycle: two tasks awaiting each other. ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "a", - [[["awaiter2"], 4], [["main"], 2]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - (4, "b", [[["awaiter"], 3]]), - ], + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], - [1, "0x3", "a", "awaiter2", "b", "0x4"], - [1, "0x3", "a", "main", "Task-1", "0x2"], - [1, "0x4", "b", "awaiter", "a", "0x3"], + [1, "0x2", "Task-1", "", "", "", "0x0"], + [1, "0x3", "a", "", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "", "main", "Task-1", "0x2"], + [1, "0x4", "b", "", "awaiter", "a", "0x3"], ] ), ], @@ -519,41 +1017,95 @@ TEST_INPUTS_TABLE = [ # this test case contains two cycles ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "A", - [[["nested", "nested", "task_b"], 4]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] ), - ( - 4, - "B", - [ - [["nested", "nested", "task_c"], 5], - [["nested", "nested", "task_a"], 3], - ], + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] ), - (5, "C", [[["nested", "nested"], 6]]), - ( - 6, - "Task-2", - [[["nested", "nested", "task_b"], 4]], + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] ), - ], + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], + [1, "0x2", "Task-1", "", "", "", "0x0"], [ 1, "0x3", "A", + "", "nested -> nested -> task_b", "B", "0x4", @@ -562,6 +1114,7 @@ TEST_INPUTS_TABLE = [ 1, "0x4", "B", + "", "nested -> nested -> task_c", "C", "0x5", @@ -570,6 +1123,7 @@ TEST_INPUTS_TABLE = [ 1, "0x4", "B", + "", "nested -> nested -> task_a", "A", "0x3", @@ -578,6 +1132,7 @@ TEST_INPUTS_TABLE = [ 1, "0x5", "C", + "", "nested -> nested", "Task-2", "0x6", @@ -586,6 +1141,7 @@ TEST_INPUTS_TABLE = [ 1, "0x6", "Task-2", + "", "nested -> nested -> task_b", "B", "0x4", @@ -600,7 +1156,8 @@ class TestAsyncioToolsTree(unittest.TestCase): def test_asyncio_utils(self): for input_, tree in TEST_INPUTS_TREE: with self.subTest(input_): - self.assertEqual(tools.build_async_tree(input_), tree) + result = tools.build_async_tree(input_) + self.assertEqual(result, tree) def test_asyncio_utils_cycles(self): for input_, cycles in TEST_INPUTS_CYCLES_TREE: @@ -615,7 +1172,8 @@ class TestAsyncioToolsTable(unittest.TestCase): def test_asyncio_utils(self): for input_, table in TEST_INPUTS_TABLE: with self.subTest(input_): - self.assertEqual(tools.build_task_table(input_), table) + result = tools.build_task_table(input_) + self.assertEqual(result, table) class TestAsyncioToolsBasic(unittest.TestCase): @@ -632,26 +1190,67 @@ class TestAsyncioToolsBasic(unittest.TestCase): self.assertEqual(tools.build_task_table(result), expected_output) def test_only_independent_tasks_tree(self): - input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] expected = [["└── (T) taskA"], ["└── (T) taskB"]] result = tools.build_async_tree(input_) self.assertEqual(sorted(result), sorted(expected)) def test_only_independent_tasks_table(self): - input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] self.assertEqual( tools.build_task_table(input_), - [[1, "0xa", "taskA", "", "", "0x0"], [1, "0xb", "taskB", "", "", "0x0"]], + [[1, '0xa', 'taskA', '', '', '', '0x0'], [1, '0xb', 'taskB', '', '', '', '0x0']] ) def test_single_task_tree(self): """Test build_async_tree with a single task and no awaits.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] expected_output = [ @@ -664,25 +1263,50 @@ class TestAsyncioToolsBasic(unittest.TestCase): def test_single_task_table(self): """Test build_task_table with a single task and no awaits.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] - expected_output = [[1, "0x2", "Task-1", "", "", "0x0"]] + expected_output = [[1, '0x2', 'Task-1', '', '', '', '0x0']] self.assertEqual(tools.build_task_table(result), expected_output) def test_cycle_detection(self): """Test build_async_tree raises CycleFoundException for cyclic input.""" result = [ - ( - 1, - [ - (2, "Task-1", [[["main"], 3]]), - (3, "Task-2", [[["main"], 2]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ) + ] ) ] with self.assertRaises(tools.CycleFoundException) as context: @@ -692,13 +1316,38 @@ class TestAsyncioToolsBasic(unittest.TestCase): def test_complex_tree(self): """Test build_async_tree with a more complex tree structure.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - (3, "Task-2", [[["main"], 2]]), - (4, "Task-3", [[["main"], 3]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] ) ] expected_output = [ @@ -715,30 +1364,76 @@ class TestAsyncioToolsBasic(unittest.TestCase): def test_complex_table(self): """Test build_task_table with a more complex tree structure.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - (3, "Task-2", [[["main"], 2]]), - (4, "Task-3", [[["main"], 3]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] ) ] expected_output = [ - [1, "0x2", "Task-1", "", "", "0x0"], - [1, "0x3", "Task-2", "main", "Task-1", "0x2"], - [1, "0x4", "Task-3", "main", "Task-2", "0x3"], + [1, '0x2', 'Task-1', '', '', '', '0x0'], + [1, '0x3', 'Task-2', '', 'main', 'Task-1', '0x2'], + [1, '0x4', 'Task-3', '', 'main', 'Task-2', '0x3'] ] self.assertEqual(tools.build_task_table(result), expected_output) def test_deep_coroutine_chain(self): input_ = [ - ( - 1, - [ - (10, "leaf", [[["c1", "c2", "c3", "c4", "c5"], 11]]), - (11, "root", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="leaf", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("c1", "", 0), + FrameInfo("c2", "", 0), + FrameInfo("c3", "", 0), + FrameInfo("c4", "", 0), + FrameInfo("c5", "", 0) + ], + task_name=11 + ) + ] + ), + TaskInfo( + task_id=11, + task_name="root", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] expected = [ @@ -757,13 +1452,47 @@ class TestAsyncioToolsBasic(unittest.TestCase): def test_multiple_cycles_same_node(self): input_ = [ - ( - 1, - [ - (1, "Task-A", [[["call1"], 2]]), - (2, "Task-B", [[["call2"], 3]]), - (3, "Task-C", [[["call3"], 1], [["call4"], 2]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call2", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call3", "", 0)], + task_name=1 + ), + CoroInfo( + call_stack=[FrameInfo("call4", "", 0)], + task_name=2 + ) + ] + ) + ] ) ] with self.assertRaises(tools.CycleFoundException) as ctx: @@ -772,19 +1501,43 @@ class TestAsyncioToolsBasic(unittest.TestCase): self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) def test_table_output_format(self): - input_ = [(1, [(1, "Task-A", [[["foo"], 2]]), (2, "Task-B", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("foo", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] table = tools.build_task_table(input_) for row in table: - self.assertEqual(len(row), 6) + self.assertEqual(len(row), 7) self.assertIsInstance(row[0], int) # thread ID self.assertTrue( isinstance(row[1], str) and row[1].startswith("0x") ) # hex task ID self.assertIsInstance(row[2], str) # task name - self.assertIsInstance(row[3], str) # coroutine chain - self.assertIsInstance(row[4], str) # awaiter name + self.assertIsInstance(row[3], str) # coroutine stack + self.assertIsInstance(row[4], str) # coroutine chain + self.assertIsInstance(row[5], str) # awaiter name self.assertTrue( - isinstance(row[5], str) and row[5].startswith("0x") + isinstance(row[6], str) and row[6].startswith("0x") ) # hex awaiter ID @@ -792,28 +1545,86 @@ class TestAsyncioToolsEdgeCases(unittest.TestCase): def test_task_awaits_self(self): """A task directly awaits itself - should raise a cycle.""" - input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Self-Awaiter", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("loopback", "", 0)], + task_name=1 + ) + ] + ) + ] + ) + ] with self.assertRaises(tools.CycleFoundException) as ctx: tools.build_async_tree(input_) self.assertIn([1, 1], ctx.exception.cycles) def test_task_with_missing_awaiter_id(self): """Awaiter ID not in task list - should not crash, just show 'Unknown'.""" - input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])] # 999 not defined + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("coro", "", 0)], + task_name=999 + ) + ] + ) + ] + ) + ] table = tools.build_task_table(input_) self.assertEqual(len(table), 1) - self.assertEqual(table[0][4], "Unknown") + self.assertEqual(table[0][5], "Unknown") def test_duplicate_coroutine_frames(self): """Same coroutine frame repeated under a parent - should deduplicate.""" input_ = [ - ( - 1, - [ - (1, "Task-1", [[["frameA"], 2], [["frameA"], 3]]), - (2, "Task-2", []), - (3, "Task-3", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=2 + ), + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] tree = tools.build_async_tree(input_) @@ -830,14 +1641,63 @@ class TestAsyncioToolsEdgeCases(unittest.TestCase): def test_task_with_no_name(self): """Task with no name in id2name - should still render with fallback.""" - input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="root", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("f1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name=None, + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] # If name is None, fallback to string should not crash tree = tools.build_async_tree(input_) self.assertIn("(T) None", "\n".join(tree[0])) def test_tree_rendering_with_custom_emojis(self): """Pass custom emojis to the tree renderer.""" - input_ = [(1, [(1, "MainTask", [[["f1", "f2"], 2]]), (2, "SubTask", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="MainTask", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("f1", "", 0), + FrameInfo("f2", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="SubTask", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") flat = "\n".join(tree[0]) self.assertIn("🧵 MainTask", flat) diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 5f9eb381f60..077765fcda2 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -322,6 +322,14 @@ class AuditTest(unittest.TestCase): if returncode: self.fail(stderr) + @support.support_remote_exec_only + @support.cpython_only + def test_sys_remote_exec(self): + returncode, events, stderr = self.run_python("test_sys_remote_exec") + self.assertTrue(any(["sys.remote_exec" in event for event in events])) + self.assertTrue(any(["cpython.remote_debugger_script" in event for event in events])) + if returncode: + self.fail(stderr) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 05ce163a337..ba4b8c5aa9b 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -117,12 +117,26 @@ class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): # Override generic format tests with tests for our specific implemenation. @needs_installed_python - @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS run tests via a custom testbed method that changes sys.executable') + @unittest.skipIf( + is_android or is_apple_mobile, + 'Android and iOS run tests via a custom testbed method that changes sys.executable' + ) def test_base_interpreter(self): value = self.key('base_interpreter') self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable)) + @needs_installed_python + @unittest.skipIf( + is_android or is_apple_mobile, + "Android and iOS run tests via a custom testbed method that doesn't ship headers" + ) + def test_c_api(self): + value = self.key('c_api') + self.assertTrue(os.path.exists(os.path.join(value['headers'], 'Python.h'))) + version = sysconfig.get_config_var('VERSION') + self.assertTrue(os.path.exists(os.path.join(value['pkgconfig_path'], f'python-{version}.pc'))) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index d221aa5e1d9..14fe3355239 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2991,7 +2991,8 @@ class TestType(unittest.TestCase): def load_tests(loader, tests, pattern): from doctest import DocTestSuite - tests.addTest(DocTestSuite(builtins)) + if sys.float_repr_style == 'short': + tests.addTest(DocTestSuite(builtins)) return tests if __name__ == "__main__": diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index f74694a7a74..ef950f5df04 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -413,11 +413,13 @@ class CAPITest(unittest.TestCase): @support.requires_resource('cpu') @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_trashcan_python_class1(self): self.do_test_trashcan_python_class(list) @support.requires_resource('cpu') @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_trashcan_python_class2(self): from _testcapi import MyList self.do_test_trashcan_python_class(MyList) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index bf22ef2a592..e4c9a463855 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -407,12 +407,12 @@ class TestUops(unittest.TestCase): x = 0 for i in range(m): for j in MyIter(n): - x += 1000*i + j + x += j return x - x = testfunc(TIER2_THRESHOLD, TIER2_THRESHOLD) + x = testfunc(TIER2_THRESHOLD, 2) - self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * TIER2_THRESHOLD * 1001) + self.assertEqual(x, sum(range(TIER2_THRESHOLD)) * 2) ex = get_first_executor(testfunc) self.assertIsNotNone(ex) @@ -678,7 +678,7 @@ class TestUopsOptimization(unittest.TestCase): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_ADD_FLOAT", uops) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) def test_float_subtract_constant_propagation(self): def testfunc(n): @@ -700,7 +700,7 @@ class TestUopsOptimization(unittest.TestCase): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_SUBTRACT_FLOAT", uops) + self.assertIn("_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", uops) def test_float_multiply_constant_propagation(self): def testfunc(n): @@ -722,7 +722,7 @@ class TestUopsOptimization(unittest.TestCase): self.assertLessEqual(len(guard_nos_float_count), 1) # TODO gh-115506: this assertion may change after propagating constants. # We'll also need to verify that propagation actually occurs. - self.assertIn("_BINARY_OP_MULTIPLY_FLOAT", uops) + self.assertIn("_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", uops) def test_add_unicode_propagation(self): def testfunc(n): @@ -1381,6 +1381,21 @@ class TestUopsOptimization(unittest.TestCase): # Removed guard self.assertNotIn("_CHECK_FUNCTION_EXACT_ARGS", uops) + def test_method_guards_removed_or_reduced(self): + def testfunc(n): + result = 0 + for i in range(n): + result += test_bound_method(i) + return result + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, sum(range(TIER2_THRESHOLD))) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_PUSH_FRAME", uops) + # Strength reduced version + self.assertIn("_CHECK_FUNCTION_VERSION_INLINE", uops) + self.assertNotIn("_CHECK_METHOD_VERSION", uops) + def test_jit_error_pops(self): """ Tests that the correct number of pops are inserted into the @@ -1953,6 +1968,49 @@ class TestUopsOptimization(unittest.TestCase): self.assertNotIn("_GUARD_NOS_INT", uops) self.assertNotIn("_GUARD_TOS_INT", uops) + def test_call_len_known_length_small_int(self): + def testfunc(n): + x = 0 + for _ in range(n): + t = (1, 2, 3, 4, 5) + if len(t) == 5: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # When the length is < _PY_NSMALLPOSINTS, the len() call is replaced + # with just an inline load. + self.assertNotIn("_CALL_LEN", uops) + self.assertNotIn("_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_CALL_LOAD_CONST_INLINE_BORROW", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_call_len_known_length(self): + def testfunc(n): + class C: + t = tuple(range(300)) + + x = 0 + for _ in range(n): + if len(C.t) == 300: # comparison + guard removed + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # When the length is >= _PY_NSMALLPOSINTS, we cannot replace + # the len() call with an inline load, but knowing the exact + # length allows us to optimize more code, such as conditionals + # in this case + self.assertIn("_CALL_LEN", uops) + self.assertNotIn("_COMPARE_OP_INT", uops) + self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + def test_get_len_with_const_tuple(self): def testfunc(n): x = 0.0 @@ -2247,6 +2305,20 @@ class TestUopsOptimization(unittest.TestCase): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_float_op_refcount_elimination(self): + def testfunc(args): + a, b, n = args + c = 0.0 + for _ in range(n): + c += a + b + return c + + res, ex = self._run_with_optimizer(testfunc, (0.1, 0.1, TIER2_THRESHOLD)) + self.assertAlmostEqual(res, TIER2_THRESHOLD * (0.1 + 0.1)) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", uops) + def test_remove_guard_for_slice_list(self): def f(n): for i in range(n): @@ -2290,9 +2362,104 @@ class TestUopsOptimization(unittest.TestCase): self.assertNotIn("_GUARD_TOS_INT", uops) self.assertNotIn("_GUARD_NOS_INT", uops) + def test_attr_promotion_failure(self): + # We're not testing for any specific uops here, just + # testing it doesn't crash. + script_helper.assert_python_ok('-c', textwrap.dedent(""" + import _testinternalcapi + import _opcode + import email + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + def testfunc(n): + for _ in range(n): + email.jit_testing = None + prompt = email.jit_testing + del email.jit_testing + + + testfunc(_testinternalcapi.TIER2_THRESHOLD) + ex = get_first_executor(testfunc) + assert ex is not None + """)) + + def test_pop_top_specialize_none(self): + def testfunc(n): + for _ in range(n): + global_identity(None) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_POP_TOP_NOP", uops) + + def test_pop_top_specialize_int(self): + def testfunc(n): + for _ in range(n): + global_identity(100000) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_POP_TOP_INT", uops) + + def test_pop_top_specialize_float(self): + def testfunc(n): + for _ in range(n): + global_identity(1e6) + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_POP_TOP_FLOAT", uops) + + + def test_unary_negative_long_float_type(self): + def testfunc(n): + for _ in range(n): + a = 9397 + f = 9397.0 + x = -a + -a + y = -f + -f + + testfunc(TIER2_THRESHOLD) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertNotIn("_GUARD_NOS_INT", uops) + self.assertNotIn("_GUARD_TOS_FLOAT", uops) + self.assertNotIn("_GUARD_NOS_FLOAT", uops) def global_identity(x): return x +class TestObject: + def test(self, *args, **kwargs): + return args[0] + +test_object = TestObject() +test_bound_method = TestObject.test.__get__(test_object) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 9fc2b047bef..655f5a9be7f 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -701,6 +701,7 @@ class CodeTest(unittest.TestCase): 'checks': CO_FAST_LOCAL, 'res': CO_FAST_LOCAL, }, + defs.spam_with_global_and_attr_same_name: {}, defs.spam_full_args: { 'a': POSONLY, 'b': POSONLY, @@ -955,6 +956,10 @@ class CodeTest(unittest.TestCase): purelocals=5, globalvars=6, ), + defs.spam_with_global_and_attr_same_name: new_var_counts( + globalvars=2, + attrs=1, + ), defs.spam_full_args: new_var_counts( posonly=2, posorkw=2, diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index df640929309..6b8484c0d5f 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -20,6 +20,10 @@ INITIALIZER_STATUS = 'uninitialized' def init(x): global INITIALIZER_STATUS INITIALIZER_STATUS = x + # InterpreterPoolInitializerTest.test_initializer fails + # if we don't have a LOAD_GLOBAL. (It could be any global.) + # We will address this separately. + INITIALIZER_STATUS def get_init_status(): return INITIALIZER_STATUS diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index f6c62ae4b20..844dfdd6fc9 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -2,35 +2,78 @@ import asyncio import contextlib import io import os -import pickle +import sys import time import unittest -from concurrent.futures.interpreter import ( - ExecutionFailed, BrokenInterpreterPool, -) +from concurrent.futures.interpreter import BrokenInterpreterPool +from concurrent import interpreters +from concurrent.interpreters import _queues as queues import _interpreters from test import support +from test.support import os_helper +from test.support import script_helper import test.test_asyncio.utils as testasyncio_utils -from test.support.interpreters import queues from .executor import ExecutorTest, mul from .util import BaseTestCase, InterpreterPoolMixin, setup_module +WINDOWS = sys.platform.startswith('win') + + +@contextlib.contextmanager +def nonblocking(fd): + blocking = os.get_blocking(fd) + if blocking: + os.set_blocking(fd, False) + try: + yield + finally: + if blocking: + os.set_blocking(fd, blocking) + + +def read_file_with_timeout(fd, nbytes, timeout): + with nonblocking(fd): + end = time.time() + timeout + try: + return os.read(fd, nbytes) + except BlockingIOError: + pass + while time.time() < end: + try: + return os.read(fd, nbytes) + except BlockingIOError: + continue + else: + raise TimeoutError('nothing to read') + + +if not WINDOWS: + import select + def read_file_with_timeout(fd, nbytes, timeout): + r, _, _ = select.select([fd], [], [], timeout) + if fd not in r: + raise TimeoutError('nothing to read') + return os.read(fd, nbytes) + + def noop(): pass def write_msg(fd, msg): + import os os.write(fd, msg + b'\0') -def read_msg(fd): +def read_msg(fd, timeout=10.0): msg = b'' - while ch := os.read(fd, 1): - if ch == b'\0': - return msg + ch = read_file_with_timeout(fd, 1, timeout) + while ch != b'\0': msg += ch + ch = os.read(fd, 1) + return msg def get_current_name(): @@ -113,6 +156,38 @@ class InterpreterPoolExecutorTest( self.assertEqual(before, b'\0') self.assertEqual(after, msg) + def test_init_with___main___global(self): + # See https://github.com/python/cpython/pull/133957#issuecomment-2927415311. + text = """if True: + from concurrent.futures import InterpreterPoolExecutor + + INITIALIZER_STATUS = 'uninitialized' + + def init(x): + global INITIALIZER_STATUS + INITIALIZER_STATUS = x + INITIALIZER_STATUS + + def get_init_status(): + return INITIALIZER_STATUS + + if __name__ == "__main__": + exe = InterpreterPoolExecutor(initializer=init, + initargs=('initialized',)) + fut = exe.submit(get_init_status) + print(fut.result()) # 'initialized' + exe.shutdown(wait=True) + print(INITIALIZER_STATUS) # 'uninitialized' + """ + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + self.assertEqual(stdout.splitlines(), [ + 'initialized', + 'uninitialized', + ]) + def test_init_closure(self): count = 0 def init1(): @@ -121,10 +196,19 @@ class InterpreterPoolExecutorTest( nonlocal count count += 1 - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=init1) - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=init2) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=init1) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() + + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=init2) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() def test_init_instance_method(self): class Spam: @@ -132,26 +216,12 @@ class InterpreterPoolExecutorTest( raise NotImplementedError spam = Spam() - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=spam.initializer) - - def test_init_shared(self): - msg = b'eggs' - r, w = self.pipe() - script = f"""if True: - import os - if __name__ != '__main__': - import __main__ - spam = __main__.spam - os.write({w}, spam + b'\\0') - """ - - executor = self.executor_type(shared={'spam': msg}) - fut = executor.submit(exec, script) - fut.result() - after = read_msg(r) - - self.assertEqual(after, msg) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=spam.initializer) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() @unittest.expectedFailure def test_init_exception_in_script(self): @@ -178,8 +248,6 @@ class InterpreterPoolExecutorTest( stderr = stderr.getvalue() self.assertIn('ExecutionFailed: Exception: spam', stderr) self.assertIn('Uncaught in the interpreter:', stderr) - self.assertIn('The above exception was the direct cause of the following exception:', - stderr) @unittest.expectedFailure def test_submit_script(self): @@ -208,10 +276,14 @@ class InterpreterPoolExecutorTest( return spam executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(task1) - with self.assertRaises(pickle.PicklingError): - executor.submit(task2) + + fut = executor.submit(task1) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() + + fut = executor.submit(task2) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_local_instance(self): class Spam: @@ -219,8 +291,9 @@ class InterpreterPoolExecutorTest( self.value = True executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(Spam) + fut = executor.submit(Spam) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_instance_method(self): class Spam: @@ -229,8 +302,9 @@ class InterpreterPoolExecutorTest( spam = Spam() executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(spam.run) + fut = executor.submit(spam.run) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_func_globals(self): executor = self.executor_type() @@ -242,13 +316,14 @@ class InterpreterPoolExecutorTest( @unittest.expectedFailure def test_submit_exception_in_script(self): + # Scripts are not supported currently. fut = self.executor.submit('raise Exception("spam")') with self.assertRaises(Exception) as captured: fut.result() self.assertIs(type(captured.exception), Exception) self.assertEqual(str(captured.exception), 'spam') cause = captured.exception.__cause__ - self.assertIs(type(cause), ExecutionFailed) + self.assertIs(type(cause), interpreters.ExecutionFailed) for attr in ('__name__', '__qualname__', '__module__'): self.assertEqual(getattr(cause.excinfo.type, attr), getattr(Exception, attr)) @@ -261,7 +336,7 @@ class InterpreterPoolExecutorTest( self.assertIs(type(captured.exception), Exception) self.assertEqual(str(captured.exception), 'spam') cause = captured.exception.__cause__ - self.assertIs(type(cause), ExecutionFailed) + self.assertIs(type(cause), interpreters.ExecutionFailed) for attr in ('__name__', '__qualname__', '__module__'): self.assertEqual(getattr(cause.excinfo.type, attr), getattr(Exception, attr)) @@ -269,16 +344,93 @@ class InterpreterPoolExecutorTest( def test_saturation(self): blocker = queues.create() - executor = self.executor_type(4, shared=dict(blocker=blocker)) + executor = self.executor_type(4) for i in range(15 * executor._max_workers): - executor.submit(exec, 'import __main__; __main__.blocker.get()') - #executor.submit('blocker.get()') + executor.submit(blocker.get) self.assertEqual(len(executor._threads), executor._max_workers) for i in range(15 * executor._max_workers): blocker.put_nowait(None) executor.shutdown(wait=True) + def test_blocking(self): + # There is no guarantee that a worker will be created for every + # submitted task. That's because there's a race between: + # + # * a new worker thread, created when task A was just submitted, + # becoming non-idle when it picks up task A + # * after task B is added to the queue, a new worker thread + # is started only if there are no idle workers + # (the check in ThreadPoolExecutor._adjust_thread_count()) + # + # That means we must not block waiting for *all* tasks to report + # "ready" before we unblock the known-ready workers. + ready = queues.create() + blocker = queues.create() + + def run(taskid, ready, blocker): + # There can't be any globals here. + ready.put_nowait(taskid) + blocker.get() # blocking + + numtasks = 10 + futures = [] + with self.executor_type() as executor: + # Request the jobs. + for i in range(numtasks): + fut = executor.submit(run, i, ready, blocker) + futures.append(fut) + pending = numtasks + while pending > 0: + # Wait for any to be ready. + done = 0 + for _ in range(pending): + try: + ready.get(timeout=1) # blocking + except interpreters.QueueEmpty: + pass + else: + done += 1 + pending -= done + # Unblock the workers. + for _ in range(done): + blocker.put_nowait(None) + + def test_blocking_with_limited_workers(self): + # This is essentially the same as test_blocking, + # but we explicitly force a limited number of workers, + # instead of it happening implicitly sometimes due to a race. + ready = queues.create() + blocker = queues.create() + + def run(taskid, ready, blocker): + # There can't be any globals here. + ready.put_nowait(taskid) + blocker.get() # blocking + + numtasks = 10 + futures = [] + with self.executor_type(4) as executor: + # Request the jobs. + for i in range(numtasks): + fut = executor.submit(run, i, ready, blocker) + futures.append(fut) + pending = numtasks + while pending > 0: + # Wait for any to be ready. + done = 0 + for _ in range(pending): + try: + ready.get(timeout=1) # blocking + except interpreters.QueueEmpty: + pass + else: + done += 1 + pending -= done + # Unblock the workers. + for _ in range(done): + blocker.put_nowait(None) + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() @@ -289,12 +441,21 @@ class InterpreterPoolExecutorTest( executor.shutdown(wait=True) def test_pickle_errors_propagate(self): - # GH-125864: Pickle errors happen before the script tries to execute, so the - # queue used to wait infinitely. - + # GH-125864: Pickle errors happen before the script tries to execute, + # so the queue used to wait infinitely. fut = self.executor.submit(PickleShenanigans(0)) - with self.assertRaisesRegex(RuntimeError, "gotcha"): + expected = interpreters.NotShareableError + with self.assertRaisesRegex(expected, 'args not shareable') as cm: fut.result() + self.assertRegex(str(cm.exception.__cause__), 'unpickled') + + def test_no_stale_references(self): + # Weak references don't cross between interpreters. + raise unittest.SkipTest('not applicable') + + def test_free_reference(self): + # Weak references don't cross between interpreters. + raise unittest.SkipTest('not applicable') class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase): diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 23904d17d32..e7364e18742 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -986,12 +986,12 @@ class ConfigParserTestCase(BasicTestCase, unittest.TestCase): def test_defaults_keyword(self): """bpo-23835 fix for ConfigParser""" - cf = self.newconfig(defaults={1: 2.4}) - self.assertEqual(cf[self.default_section]['1'], '2.4') - self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.4) - cf = self.newconfig(defaults={"A": 5.2}) - self.assertEqual(cf[self.default_section]['a'], '5.2') - self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.2) + cf = self.newconfig(defaults={1: 2.5}) + self.assertEqual(cf[self.default_section]['1'], '2.5') + self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.5) + cf = self.newconfig(defaults={"A": 5.25}) + self.assertEqual(cf[self.default_section]['a'], '5.25') + self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.25) class ConfigParserTestCaseNoInterpolation(BasicTestCase, unittest.TestCase): diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 192c8eab26e..57e818b1c68 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -125,21 +125,22 @@ class CProfileTest(ProfileTest): """ gh-106152 generator.throw() should trigger a call in cProfile - In the any() call below, there should be two entries for the generator: - * one for the call to __next__ which gets a True and terminates any - * one when the generator is garbage collected which will effectively - do a throw. """ + + def gen(): + yield + pr = self.profilerclass() pr.enable() - any(a == 1 for a in (1, 2)) + g = gen() + try: + g.throw(SyntaxError) + except SyntaxError: + pass pr.disable() pr.create_stats() - for func, (cc, nc, _, _, _) in pr.stats.items(): - if func[2] == "<genexpr>": - self.assertEqual(cc, 1) - self.assertEqual(nc, 1) + self.assertTrue(any("throw" in func[2] for func in pr.stats.keys())), def test_bad_descriptor(self): # gh-132250 diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index f89521cf8b3..46f8ff93efa 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,3 +1,4 @@ +import sys import unittest import test.support from ctypes import (CDLL, PyDLL, ArgumentError, @@ -240,7 +241,8 @@ class SimpleTypesTestCase(unittest.TestCase): self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^<cparam '[LIQ]' \(20000\)>$") self.assertEqual(repr(c_float.from_param(1.5)), "<cparam 'f' (1.5)>") self.assertEqual(repr(c_double.from_param(1.5)), "<cparam 'd' (1.5)>") - self.assertEqual(repr(c_double.from_param(1e300)), "<cparam 'd' (1e+300)>") + if sys.float_repr_style == 'short': + self.assertEqual(repr(c_double.from_param(1e300)), "<cparam 'd' (1e+300)>") self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^<cparam ('d' \(1.5\)|'g' at 0x[A-Fa-f0-9]+)>$") self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^<cparam 'z' \(0x[A-Fa-f0-9]+\)>$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^<cparam 'Z' \(0x[A-Fa-f0-9]+\)>$") diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 52c38e42eca..60c62430370 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -290,6 +290,38 @@ class DictTest(unittest.TestCase): ['Cannot convert dictionary update sequence element #0 to a sequence'], ) + def test_update_shared_keys(self): + class MyClass: pass + + # Subclass str to enable us to create an object during the + # dict.update() call. + class MyStr(str): + def __hash__(self): + return super().__hash__() + + def __eq__(self, other): + # Create an object that shares the same PyDictKeysObject as + # obj.__dict__. + obj2 = MyClass() + obj2.a = "a" + obj2.b = "b" + obj2.c = "c" + return super().__eq__(other) + + obj = MyClass() + obj.a = "a" + obj.b = "b" + + x = {} + x[MyStr("a")] = MyStr("a") + + # gh-132617: this previously raised "dict mutated during update" error + x.update(obj.__dict__) + + self.assertEqual(x, { + MyStr("a"): "a", + "b": "b", + }) def test_fromkeys(self): self.assertEqual(dict.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 8765d121fd0..b8116d073a2 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -2568,6 +2568,18 @@ Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar =?mac-iceland?q?r=8Aksm?= self.assertEqual(str(make_header(decode_header(s))), '"Müller T" <T.Mueller@xxx.com>') + def test_unencoded_ascii(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header without encoded words' + self.assertEqual(decode_header(s), + [('header without encoded words', None)]) + + def test_unencoded_utf8(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header with unexpected non ASCII caract\xe8res' + self.assertEqual(decode_header(s), + [('header with unexpected non ASCII caract\xe8res', None)]) + # Test the MIMEMessage class class TestMIMEMessage(TestEmailBase): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 221f9db7763..bbc7630fa83 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -36,7 +36,7 @@ def load_tests(loader, tests, ignore): optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) howto_tests = os.path.join(REPO_ROOT, 'Doc/howto/enum.rst') - if os.path.exists(howto_tests): + if os.path.exists(howto_tests) and sys.float_repr_style == 'short': tests.addTests(doctest.DocFileSuite( howto_tests, module_relative=False, diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 175ef531386..57d0656487d 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1445,6 +1445,7 @@ class ExceptionTests(unittest.TestCase): foo() support.gc_collect() + @support.skip_emscripten_stack_overflow() @cpython_only def test_recursion_normalizing_exception(self): import_module("_testinternalcapi") @@ -1522,6 +1523,7 @@ class ExceptionTests(unittest.TestCase): self.assertIn(b'Done.', out) + @support.skip_emscripten_stack_overflow() def test_recursion_in_except_handler(self): def set_relative_recursion_limit(n): diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 303af25fc7a..90214e814f2 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -5,7 +5,7 @@ import importlib import sys import socket import threading -from asyncio import staggered, taskgroups +from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY from test.support import os_helper, SHORT_TIMEOUT, busy_retry from test.support.script_helper import make_script @@ -18,8 +18,12 @@ PROCESS_VM_READV_SUPPORTED = False try: from _remote_debugging import PROCESS_VM_READV_SUPPORTED from _remote_debugging import RemoteUnwinder + from _remote_debugging import FrameInfo, CoroInfo, TaskInfo except ImportError: - raise unittest.SkipTest("Test only runs when _remote_debugging is available") + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) @@ -28,7 +32,11 @@ def _make_test_script(script_dir, script_basename, source): skip_if_not_supported = unittest.skipIf( - (sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32"), + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), "Test only runs on Linux, Windows and MacOS", ) @@ -101,11 +109,16 @@ class TestGetStackTrace(unittest.TestCase): client_socket, _ = server_socket.accept() server_socket.close() response = b"" - while b"ready:main" not in response or b"ready:thread" not in response: + while ( + b"ready:main" not in response + or b"ready:thread" not in response + ): response += client_socket.recv(1024) stack_trace = get_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -114,17 +127,17 @@ class TestGetStackTrace(unittest.TestCase): p.wait(timeout=SHORT_TIMEOUT) thread_expected_stack_trace = [ - (script_name, 15, "foo"), - (script_name, 12, "baz"), - (script_name, 9, "bar"), - (threading.__file__, ANY, 'Thread.run') + FrameInfo([script_name, 15, "foo"]), + FrameInfo([script_name, 12, "baz"]), + FrameInfo([script_name, 9, "bar"]), + FrameInfo([threading.__file__, ANY, "Thread.run"]), ] # Is possible that there are more threads, so we check that the # expected stack traces are in the result (looking at you Windows!) self.assertIn((ANY, thread_expected_stack_trace), stack_trace) # Check that the main thread stack trace is in the result - frame = (script_name, 19, "<module>") + frame = FrameInfo([script_name, 19, "<module>"]) for _, stack in stack_trace: if frame in stack: break @@ -189,8 +202,12 @@ class TestGetStackTrace(unittest.TestCase): ): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM + ) + server_socket.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 + ) server_socket.bind(("localhost", port)) server_socket.settimeout(SHORT_TIMEOUT) server_socket.listen(1) @@ -208,7 +225,9 @@ class TestGetStackTrace(unittest.TestCase): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -219,79 +238,49 @@ class TestGetStackTrace(unittest.TestCase): # sets are unordered, so we want to sort "awaited_by"s stack_trace[2].sort(key=lambda x: x[1]) - root_task = "Task-1" expected_stack_trace = [ [ - (script_name, 10, "c5"), - (script_name, 14, "c4"), - (script_name, 17, "c3"), - (script_name, 20, "c2"), + FrameInfo([script_name, 10, "c5"]), + FrameInfo([script_name, 14, "c4"]), + FrameInfo([script_name, 17, "c3"]), + FrameInfo([script_name, 20, "c2"]), ], "c2_root", [ - [ - [ - ( - taskgroups.__file__, - ANY, - "TaskGroup._aexit" - ), - ( - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__" - ), - (script_name, 26, "main"), - ], - "Task-1", - [], - ], - [ - [(script_name, 23, "c1")], - "sub_main_1", + CoroInfo( [ [ - [ - ( + FrameInfo( + [ taskgroups.__file__, ANY, - "TaskGroup._aexit" - ), - ( + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ taskgroups.__file__, ANY, - "TaskGroup.__aexit__" - ), - (script_name, 26, "main"), - ], - "Task-1", - [], - ] - ], - ], - [ - [(script_name, 23, "c1")], - "sub_main_2", + "TaskGroup.__aexit__", + ] + ), + FrameInfo([script_name, 26, "main"]), + ], + "Task-1", + ] + ), + CoroInfo( [ - [ - [ - ( - taskgroups.__file__, - ANY, - "TaskGroup._aexit" - ), - ( - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__" - ), - (script_name, 26, "main"), - ], - "Task-1", - [], - ] - ], - ], + [FrameInfo([script_name, 23, "c1"])], + "sub_main_1", + ] + ), + CoroInfo( + [ + [FrameInfo([script_name, 23, "c1"])], + "sub_main_2", + ] + ), ], ] self.assertEqual(stack_trace, expected_stack_trace) @@ -350,7 +339,9 @@ class TestGetStackTrace(unittest.TestCase): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -363,9 +354,9 @@ class TestGetStackTrace(unittest.TestCase): expected_stack_trace = [ [ - (script_name, 10, "gen_nested_call"), - (script_name, 16, "gen"), - (script_name, 19, "main"), + FrameInfo([script_name, 10, "gen_nested_call"]), + FrameInfo([script_name, 16, "gen"]), + FrameInfo([script_name, 19, "main"]), ], "Task-1", [], @@ -427,7 +418,9 @@ class TestGetStackTrace(unittest.TestCase): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -439,9 +432,12 @@ class TestGetStackTrace(unittest.TestCase): stack_trace[2].sort(key=lambda x: x[1]) expected_stack_trace = [ - [(script_name, 11, "deep"), (script_name, 15, "c1")], + [ + FrameInfo([script_name, 11, "deep"]), + FrameInfo([script_name, 15, "c1"]), + ], "Task-2", - [[[(script_name, 21, "main")], "Task-1", []]], + [CoroInfo([[FrameInfo([script_name, 21, "main"])], "Task-1"])], ] self.assertEqual(stack_trace, expected_stack_trace) @@ -503,7 +499,9 @@ class TestGetStackTrace(unittest.TestCase): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -515,20 +513,29 @@ class TestGetStackTrace(unittest.TestCase): stack_trace[2].sort(key=lambda x: x[1]) expected_stack_trace = [ [ - (script_name, 11, "deep"), - (script_name, 15, "c1"), - (staggered.__file__, ANY, "staggered_race.<locals>.run_one_coro"), + FrameInfo([script_name, 11, "deep"]), + FrameInfo([script_name, 15, "c1"]), + FrameInfo( + [ + staggered.__file__, + ANY, + "staggered_race.<locals>.run_one_coro", + ] + ), ], "Task-2", [ - [ + CoroInfo( [ - (staggered.__file__, ANY, "staggered_race"), - (script_name, 21, "main"), - ], - "Task-1", - [], - ] + [ + FrameInfo( + [staggered.__file__, ANY, "staggered_race"] + ), + FrameInfo([script_name, 21, "main"]), + ], + "Task-1", + ] + ) ], ] self.assertEqual(stack_trace, expected_stack_trace) @@ -659,62 +666,174 @@ class TestGetStackTrace(unittest.TestCase): # expected: at least 1000 pending tasks self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure - self.assertIn((ANY, "Task-1", []), entries) main_stack = [ - ( - taskgroups.__file__, - ANY, - "TaskGroup._aexit", + FrameInfo([taskgroups.__file__, ANY, "TaskGroup._aexit"]), + FrameInfo( + [taskgroups.__file__, ANY, "TaskGroup.__aexit__"] ), - ( - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ), - (script_name, 60, "main"), + FrameInfo([script_name, 60, "main"]), ] self.assertIn( - (ANY, "server task", [[main_stack, ANY]]), + TaskInfo( + [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []] + ), entries, ) self.assertIn( - (ANY, "echo client spam", [[main_stack, ANY]]), + TaskInfo( + [ + ANY, + "server task", + [ + CoroInfo( + [ + [ + FrameInfo( + [ + base_events.__file__, + ANY, + "Server.serve_forever", + ] + ) + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, ANY, "main"] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) + self.assertIn( + TaskInfo( + [ + ANY, + "Task-4", + [ + CoroInfo( + [ + [ + FrameInfo( + [tasks.__file__, ANY, "sleep"] + ), + FrameInfo( + [ + script_name, + 38, + "echo_client", + ] + ), + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [ + script_name, + 41, + "echo_client_spam", + ] + ), + ], + ANY, + ] + ) + ], + ] + ), entries, ) - expected_stack = [ - [ + expected_awaited_by = [ + CoroInfo( [ - ( - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ), - ( - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ), - (script_name, 41, "echo_client_spam"), - ], - ANY, - ] + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, 41, "echo_client_spam"] + ), + ], + ANY, + ] + ) ] - tasks_with_stack = [ - task for task in entries if task[2] == expected_stack + tasks_with_awaited = [ + task + for task in entries + if task.awaited_by == expected_awaited_by ] - self.assertGreaterEqual(len(tasks_with_stack), 1000) + self.assertGreaterEqual(len(tasks_with_awaited), 1000) # the final task will have some random number, but it should for # sure be one of the echo client spam horde (In windows this is not true # for some reason) if sys.platform != "win32": self.assertEqual( - expected_stack, - entries[-1][2], + tasks_with_awaited[-1].awaited_by, + entries[-1].awaited_by, ) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -740,17 +859,21 @@ class TestGetStackTrace(unittest.TestCase): self.assertEqual( stack[:2], [ - ( - __file__, - get_stack_trace.__code__.co_firstlineno + 2, - "get_stack_trace", + FrameInfo( + [ + __file__, + get_stack_trace.__code__.co_firstlineno + 2, + "get_stack_trace", + ] ), - ( - __file__, - self.test_self_trace.__code__.co_firstlineno + 6, - "TestGetStackTrace.test_self_trace", + FrameInfo( + [ + __file__, + self.test_self_trace.__code__.co_firstlineno + 6, + "TestGetStackTrace.test_self_trace", + ] ), - ] + ], ) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 237d7b5d35e..00518abcb11 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -795,6 +795,8 @@ class FormatTestCase(unittest.TestCase): self.assertRaises(ValueError, format, x, '.6,n') @support.requires_IEEE_754 + @unittest.skipUnless(sys.float_repr_style == 'short', + "applies only when using short float repr style") def test_format_testfile(self): with open(format_testfile, encoding="utf-8") as testfile: for line in testfile: diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index c7cc32e0949..1f626d87fa6 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -346,12 +346,12 @@ class FormatTest(unittest.TestCase): testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') - testcommon(b"%a", 3.14, b"3.14") + testcommon(b"%a", 3.25, b"3.25") testcommon(b"%a", b"ghi", b"b'ghi'") testcommon(b"%a", "jkl", b"'jkl'") testcommon(b"%a", "\u0544", b"'\\u0544'") # %r is an alias for %a - testcommon(b"%r", 3.14, b"3.14") + testcommon(b"%r", 3.25, b"3.25") testcommon(b"%r", b"ghi", b"b'ghi'") testcommon(b"%r", "jkl", b"'jkl'") testcommon(b"%r", "\u0544", b"'\\u0544'") @@ -407,19 +407,19 @@ class FormatTest(unittest.TestCase): self.assertEqual(format("abc", "\u2007<5"), "abc\u2007\u2007") self.assertEqual(format(123, "\u2007<5"), "123\u2007\u2007") - self.assertEqual(format(12.3, "\u2007<6"), "12.3\u2007\u2007") + self.assertEqual(format(12.5, "\u2007<6"), "12.5\u2007\u2007") self.assertEqual(format(0j, "\u2007<4"), "0j\u2007\u2007") self.assertEqual(format(1+2j, "\u2007<8"), "(1+2j)\u2007\u2007") self.assertEqual(format("abc", "\u2007>5"), "\u2007\u2007abc") self.assertEqual(format(123, "\u2007>5"), "\u2007\u2007123") - self.assertEqual(format(12.3, "\u2007>6"), "\u2007\u200712.3") + self.assertEqual(format(12.5, "\u2007>6"), "\u2007\u200712.5") self.assertEqual(format(1+2j, "\u2007>8"), "\u2007\u2007(1+2j)") self.assertEqual(format(0j, "\u2007>4"), "\u2007\u20070j") self.assertEqual(format("abc", "\u2007^5"), "\u2007abc\u2007") self.assertEqual(format(123, "\u2007^5"), "\u2007123\u2007") - self.assertEqual(format(12.3, "\u2007^6"), "\u200712.3\u2007") + self.assertEqual(format(12.5, "\u2007^6"), "\u200712.5\u2007") self.assertEqual(format(1+2j, "\u2007^8"), "\u2007(1+2j)\u2007") self.assertEqual(format(0j, "\u2007^4"), "\u20070j\u2007") diff --git a/Lib/test/test_free_threading/test_generators.py b/Lib/test/test_free_threading/test_generators.py new file mode 100644 index 00000000000..d01675eb38b --- /dev/null +++ b/Lib/test/test_free_threading/test_generators.py @@ -0,0 +1,51 @@ +import concurrent.futures +import unittest +from threading import Barrier +from unittest import TestCase +import random +import time + +from test.support import threading_helper, Py_GIL_DISABLED + +threading_helper.requires_working_threading(module=True) + + +def random_sleep(): + delay_us = random.randint(50, 100) + time.sleep(delay_us * 1e-6) + +def random_string(): + return ''.join(random.choice('0123456789ABCDEF') for _ in range(10)) + +def set_gen_name(g, b): + b.wait() + random_sleep() + g.__name__ = random_string() + return g.__name__ + +def set_gen_qualname(g, b): + b.wait() + random_sleep() + g.__qualname__ = random_string() + return g.__qualname__ + + +@unittest.skipUnless(Py_GIL_DISABLED, "Enable only in FT build") +class TestFTGenerators(TestCase): + NUM_THREADS = 4 + + def concurrent_write_with_func(self, func): + gen = (x for x in range(42)) + for j in range(1000): + with concurrent.futures.ThreadPoolExecutor(max_workers=self.NUM_THREADS) as executor: + b = Barrier(self.NUM_THREADS) + futures = {executor.submit(func, gen, b): i for i in range(self.NUM_THREADS)} + for fut in concurrent.futures.as_completed(futures): + gen_name = fut.result() + self.assertEqual(len(gen_name), 10) + + def test_concurrent_write(self): + with self.subTest(func=set_gen_name): + self.concurrent_write_with_func(func=set_gen_name) + with self.subTest(func=set_gen_qualname): + self.concurrent_write_with_func(func=set_gen_qualname) diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index f75fb264c8a..ee7adfb2b78 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -3,7 +3,7 @@ import unittest import heapq from enum import Enum -from threading import Thread, Barrier +from threading import Thread, Barrier, Lock from random import shuffle, randint from test.support import threading_helper @@ -178,6 +178,33 @@ class TestHeapq(unittest.TestCase): self.assertEqual(len(max_heap), OBJECT_COUNT) self.test_heapq.check_max_invariant(max_heap) + def test_lock_free_list_read(self): + n, n_threads = 1_000, 10 + l = [] + barrier = Barrier(n_threads * 2) + + count = 0 + lock = Lock() + + def worker(): + with lock: + nonlocal count + x = count + count += 1 + + barrier.wait() + for i in range(n): + if x % 2: + heapq.heappush(l, 1) + heapq.heappop(l) + else: + try: + l[0] + except IndexError: + pass + + self.run_concurrently(worker, (), n_threads * 2) + @staticmethod def is_sorted_ascending(lst): """ diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py index 8360afbf78c..b8663ade1d4 100644 --- a/Lib/test/test_free_threading/test_itertools.py +++ b/Lib/test/test_free_threading/test_itertools.py @@ -44,7 +44,10 @@ class ItertoolsThreading(unittest.TestCase): def work(it): barrier.wait() for _ in range(number_of_cycles): - _ = next(it) + try: + next(it) + except StopIteration: + pass data = (1, 2, 3, 4) for it in range(number_of_iterations): diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index dd58e032a8b..58a30c8e6ac 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1336,9 +1336,9 @@ x = ( def test_conversions(self): self.assertEqual(f'{3.14:10.10}', ' 3.14') - self.assertEqual(f'{3.14!s:10.10}', '3.14 ') - self.assertEqual(f'{3.14!r:10.10}', '3.14 ') - self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + self.assertEqual(f'{1.25!s:10.10}', '1.25 ') + self.assertEqual(f'{1.25!r:10.10}', '1.25 ') + self.assertEqual(f'{1.25!a:10.10}', '1.25 ') self.assertEqual(f'{"a"}', 'a') self.assertEqual(f'{"a"!r}', "'a'") @@ -1347,7 +1347,7 @@ x = ( # Conversions can have trailing whitespace after them since it # does not provide any significance self.assertEqual(f"{3!s }", "3") - self.assertEqual(f'{3.14!s :10.10}', '3.14 ') + self.assertEqual(f'{1.25!s :10.10}', '1.25 ') # Not a conversion. self.assertEqual(f'{"a!r"}', "a!r") @@ -1380,7 +1380,7 @@ x = ( for conv in ' s', ' s ': self.assertAllRaise(SyntaxError, "f-string: conversion type must come right after the" - " exclamanation mark", + " exclamation mark", ["f'{3!" + conv + "}'"]) self.assertAllRaise(SyntaxError, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 37046d8e1c0..eb01328b6ea 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -56,14 +56,14 @@ class TestEffects(unittest.TestCase): def test_effect_sizes(self): stack = Stack() inputs = [ - x := StackItem("x", None, "1"), - y := StackItem("y", None, "oparg"), - z := StackItem("z", None, "oparg*2"), + x := StackItem("x", "1"), + y := StackItem("y", "oparg"), + z := StackItem("z", "oparg*2"), ] outputs = [ - StackItem("x", None, "1"), - StackItem("b", None, "oparg*4"), - StackItem("c", None, "1"), + StackItem("x", "1"), + StackItem("b", "oparg*4"), + StackItem("c", "1"), ] null = CWriter.null() stack.pop(z, null) @@ -1103,32 +1103,6 @@ class TestGeneratedCases(unittest.TestCase): """ self.run_cases_test(input, output) - def test_pointer_to_stackref(self): - input = """ - inst(OP, (arg: _PyStackRef * -- out)) { - out = *arg; - DEAD(arg); - } - """ - output = """ - TARGET(OP) { - #if Py_TAIL_CALL_INTERP - int opcode = OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef *arg; - _PyStackRef out; - arg = (_PyStackRef *)stack_pointer[-1].bits; - out = *arg; - stack_pointer[-1] = out; - DISPATCH(); - } - """ - self.run_cases_test(input, output) - def test_unused_cached_value(self): input = """ op(FIRST, (arg1 -- out)) { @@ -2002,8 +1976,8 @@ class TestGeneratedAbstractCases(unittest.TestCase): """ output = """ case OP: { - JitOptSymbol *arg1; - JitOptSymbol *out; + JitOptRef arg1; + JitOptRef out; arg1 = stack_pointer[-1]; out = EGGS(arg1); stack_pointer[-1] = out; @@ -2011,7 +1985,7 @@ class TestGeneratedAbstractCases(unittest.TestCase): } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; @@ -2036,14 +2010,14 @@ class TestGeneratedAbstractCases(unittest.TestCase): """ output = """ case OP: { - JitOptSymbol *out; + JitOptRef out; out = sym_new_not_null(ctx); stack_pointer[-1] = out; break; } case OP2: { - JitOptSymbol *out; + JitOptRef out; out = NULL; stack_pointer[-1] = out; break; @@ -2177,7 +2151,7 @@ class TestGeneratedAbstractCases(unittest.TestCase): """ output = """ case OP: { - JitOptSymbol *foo; + JitOptRef foo; foo = NULL; stack_pointer[0] = foo; stack_pointer += 1; @@ -2250,5 +2224,202 @@ class TestGeneratedAbstractCases(unittest.TestCase): "Inputs must have equal sizes"): self.run_cases_test(input, input2, output) + def test_pure_uop_body_copied_in(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + res = PyStackRef_IsNone(foo); + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + res_stackref = PyStackRef_IsNone(foo); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in_deopt(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + DEOPT_IF(PyStackRef_IsNull(foo)); + res = foo; + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = foo; + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (PyStackRef_IsNull(foo)) { + ctx->done = true; + break; + } + res_stackref = foo; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = foo; + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_pure_uop_body_copied_in_error_if(self): + # Note: any non-escaping call works. + # In this case, we use PyStackRef_IsNone. + input = """ + pure op(OP, (foo -- res)) { + ERROR_IF(PyStackRef_IsNull(foo)); + res = foo; + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = foo; + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (PyStackRef_IsNull(foo)) { + goto error; + } + res_stackref = foo; + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = foo; + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + + def test_replace_opcode_uop_body_copied_in_complex(self): + input = """ + pure op(OP, (foo -- res)) { + if (foo) { + res = PyStackRef_IsNone(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = sym_new_known(ctx, foo); + } + """ + output = """ + case OP: { + JitOptRef foo; + JitOptRef res; + foo = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, foo) + ) { + JitOptRef foo_sym = foo; + _PyStackRef foo = sym_get_const_as_stackref(ctx, foo_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + if (foo) { + res_stackref = PyStackRef_IsNone(foo); + } + else { + res_stackref = 1; + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + stack_pointer[-1] = res; + break; + } + res = sym_new_known(ctx, foo); + stack_pointer[-1] = res; + break; + } + """ + self.run_cases_test(input, input2, output) + + def test_replace_opocode_uop_reject_array_effects(self): + input = """ + pure op(OP, (foo[2] -- res)) { + if (foo) { + res = PyStackRef_IsNone(foo); + } + else { + res = 1; + } + } + """ + input2 = """ + op(OP, (foo[2] -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(foo); + res = sym_new_unknown(ctx); + } + """ + output = """ + """ + with self.assertRaisesRegex(SyntaxError, + "Pure evaluation cannot take array-like inputs"): + self.run_cases_test(input, input2, output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index b83ae181718..5bad483ae9d 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -208,6 +208,11 @@ class HashLibTestCase(unittest.TestCase): return itertools.chain.from_iterable(constructors) @property + def shake_constructors(self): + for shake_name in self.shakes: + yield from self.constructors_to_test.get(shake_name, ()) + + @property def is_fips_mode(self): return get_fips_mode() @@ -274,7 +279,10 @@ class HashLibTestCase(unittest.TestCase): with self.assertWarnsRegex(DeprecationWarning, DEPRECATED_STRING_PARAMETER): hashlib.new(digest_name, string=b'') - if self._hashlib: + # Make sure that _hashlib contains the constructor + # to test when using a combination of libcrypto and + # interned hash implementations. + if self._hashlib and digest_name in self._hashlib._constructors: self._hashlib.new(digest_name, b'') self._hashlib.new(digest_name, data=b'') with self.assertWarnsRegex(DeprecationWarning, @@ -328,7 +336,8 @@ class HashLibTestCase(unittest.TestCase): with self.subTest(digest_name, args=args, kwds=kwds): with self.assertRaisesRegex(TypeError, errmsg): hashlib.new(digest_name, *args, **kwds) - if self._hashlib: + if (self._hashlib and + digest_name in self._hashlib._constructors): with self.assertRaisesRegex(TypeError, errmsg): self._hashlib.new(digest_name, *args, **kwds) @@ -366,6 +375,16 @@ class HashLibTestCase(unittest.TestCase): self.assertIs(constructor, _md5.md5) self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5']) + def test_copy(self): + for cons in self.hash_constructors: + h1 = cons(os.urandom(16), usedforsecurity=False) + h2 = h1.copy() + self.assertIs(type(h1), type(h2)) + self.assertEqual(h1.name, h2.name) + size = (16,) if h1.name in self.shakes else () + self.assertEqual(h1.digest(*size), h2.digest(*size)) + self.assertEqual(h1.hexdigest(*size), h2.hexdigest(*size)) + def test_hexdigest(self): for cons in self.hash_constructors: h = cons(usedforsecurity=False) @@ -376,21 +395,50 @@ class HashLibTestCase(unittest.TestCase): self.assertIsInstance(h.digest(), bytes) self.assertEqual(hexstr(h.digest()), h.hexdigest()) - def test_digest_length_overflow(self): - # See issue #34922 - large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10) - for cons in self.hash_constructors: - h = cons(usedforsecurity=False) - if h.name not in self.shakes: - continue - if HASH is not None and isinstance(h, HASH): - # _hashopenssl's take a size_t - continue - for digest in h.digest, h.hexdigest: - self.assertRaises(ValueError, digest, -10) - for length in large_sizes: - with self.assertRaises((ValueError, OverflowError)): - digest(length) + def test_shakes_zero_digest_length(self): + for constructor in self.shake_constructors: + with self.subTest(constructor=constructor): + h = constructor(b'abcdef', usedforsecurity=False) + self.assertEqual(h.digest(0), b'') + self.assertEqual(h.hexdigest(0), '') + + def test_shakes_invalid_digest_length(self): + # See https://github.com/python/cpython/issues/79103. + for constructor in self.shake_constructors: + with self.subTest(constructor=constructor): + h = constructor(usedforsecurity=False) + # Note: digest() and hexdigest() take a signed input and + # raise if it is negative; the rationale is that we use + # internally PyBytes_FromStringAndSize() and _Py_strhex() + # which both take a Py_ssize_t. + for negative_size in (-1, -10, -(1 << 31), -sys.maxsize): + self.assertRaises(ValueError, h.digest, negative_size) + self.assertRaises(ValueError, h.hexdigest, negative_size) + + def test_shakes_overflow_digest_length(self): + # See https://github.com/python/cpython/issues/135759. + + exc_types = (OverflowError, ValueError) + # HACL* accepts an 'uint32_t' while OpenSSL accepts a 'size_t'. + openssl_overflown_sizes = (sys.maxsize + 1, 2 * sys.maxsize) + # https://github.com/python/cpython/issues/79103 restricts + # the accepted built-in lengths to 2 ** 29, even if OpenSSL + # accepts such lengths. + builtin_overflown_sizes = openssl_overflown_sizes + ( + 2 ** 29, 2 ** 32 - 10, 2 ** 32, 2 ** 32 + 10, + 2 ** 61, 2 ** 64 - 10, 2 ** 64, 2 ** 64 + 10, + ) + + for constructor in self.shake_constructors: + with self.subTest(constructor=constructor): + h = constructor(usedforsecurity=False) + if HASH is not None and isinstance(h, HASH): + overflown_sizes = openssl_overflown_sizes + else: + overflown_sizes = builtin_overflown_sizes + for invalid_size in overflown_sizes: + self.assertRaises(exc_types, h.digest, invalid_size) + self.assertRaises(exc_types, h.hexdigest, invalid_size) def test_name_attribute(self): for cons in self.hash_constructors: @@ -1009,49 +1057,67 @@ class HashLibTestCase(unittest.TestCase): def test_sha256_gil(self): gil_minsize = hashlib_helper.find_gil_minsize(['_sha2', '_hashlib']) + data = b'1' + b'#' * gil_minsize + b'1' + expected = hashlib.sha256(data).hexdigest() + m = hashlib.sha256() m.update(b'1') m.update(b'#' * gil_minsize) m.update(b'1') - self.assertEqual( - m.hexdigest(), - '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94' - ) + self.assertEqual(m.hexdigest(), expected) - m = hashlib.sha256(b'1' + b'#' * gil_minsize + b'1') - self.assertEqual( - m.hexdigest(), - '1cfceca95989f51f658e3f3ffe7f1cd43726c9e088c13ee10b46f57cef135b94' - ) + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_threaded_hashing_fast(self): + # Same as test_threaded_hashing_slow() but only tests some functions + # since otherwise test_hashlib.py becomes too slow during development. + for name in ['md5', 'sha1', 'sha256', 'sha3_256', 'blake2s']: + if constructor := getattr(hashlib, name, None): + with self.subTest(name): + self.do_test_threaded_hashing(constructor, is_shake=False) + if shake_128 := getattr(hashlib, 'shake_128', None): + self.do_test_threaded_hashing(shake_128, is_shake=True) + @requires_resource('cpu') @threading_helper.reap_threads @threading_helper.requires_working_threading() - def test_threaded_hashing(self): + def test_threaded_hashing_slow(self): + for algorithm, constructors in self.constructors_to_test.items(): + is_shake = algorithm in self.shakes + for constructor in constructors: + with self.subTest(constructor.__name__, is_shake=is_shake): + self.do_test_threaded_hashing(constructor, is_shake) + + def do_test_threaded_hashing(self, constructor, is_shake): # Updating the same hash object from several threads at once # using data chunk sizes containing the same byte sequences. # # If the internal locks are working to prevent multiple # updates on the same object from running at once, the resulting # hash will be the same as doing it single threaded upfront. - hasher = hashlib.sha1() - num_threads = 5 - smallest_data = b'swineflu' - data = smallest_data * 200000 - expected_hash = hashlib.sha1(data*num_threads).hexdigest() - - def hash_in_chunks(chunk_size): - index = 0 - while index < len(data): - hasher.update(data[index:index + chunk_size]) - index += chunk_size + + # The data to hash has length s|M|q^N and the chunk size for the i-th + # thread is s|M|q^(N-i), where N is the number of threads, M is a fixed + # message of small length, and s >= 1 and q >= 2 are small integers. + smallest_size, num_threads, s, q = 8, 5, 2, 10 + + smallest_data = os.urandom(smallest_size) + data = s * smallest_data * (q ** num_threads) + + h1 = constructor(usedforsecurity=False) + h2 = constructor(data * num_threads, usedforsecurity=False) + + def update(chunk_size): + for index in range(0, len(data), chunk_size): + h1.update(data[index:index + chunk_size]) threads = [] - for threadnum in range(num_threads): - chunk_size = len(data) // (10 ** threadnum) + for thread_num in range(num_threads): + # chunk_size = len(data) // (q ** thread_num) + chunk_size = s * smallest_size * q ** (num_threads - thread_num) self.assertGreater(chunk_size, 0) - self.assertEqual(chunk_size % len(smallest_data), 0) - thread = threading.Thread(target=hash_in_chunks, - args=(chunk_size,)) + self.assertEqual(chunk_size % smallest_size, 0) + thread = threading.Thread(target=update, args=(chunk_size,)) threads.append(thread) for thread in threads: @@ -1059,7 +1125,10 @@ class HashLibTestCase(unittest.TestCase): for thread in threads: thread.join() - self.assertEqual(expected_hash, hasher.hexdigest()) + if is_shake: + self.assertEqual(h1.hexdigest(16), h2.hexdigest(16)) + else: + self.assertEqual(h1.hexdigest(), h2.hexdigest()) def test_get_fips_mode(self): fips_mode = self.is_fips_mode diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 61fa24fab57..65a4bee72b9 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -5,6 +5,7 @@ import pprint import unittest from unittest.mock import patch +from test import support class EventCollector(html.parser.HTMLParser): @@ -430,28 +431,34 @@ text ('data', '<'), ('starttag', 'bc<', [('a', None)]), ('endtag', 'html'), - ('data', '\n<img src="URL>'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("</>", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) - self._run_check("</a", [('data', '</a')]) + self._run_check("</a", []) + self._run_check("</ a>", [('endtag', 'a')]) + self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) - self._run_check("<!", [('data', '<!')]) - self._run_check("<a", [('data', '<a')]) - self._run_check("<a foo='bar'", [('data', "<a foo='bar'")]) - self._run_check("<a foo='bar", [('data', "<a foo='bar")]) - self._run_check("<a foo='>'", [('data', "<a foo='>'")]) - self._run_check("<a foo='>", [('data', "<a foo='>")]) + self._run_check("<!", [('comment', '')]) + self._run_check("<a", []) + self._run_check("<a foo='bar'", []) + self._run_check("<a foo='bar", []) + self._run_check("<a foo='>'", []) + self._run_check("<a foo='>", []) self._run_check("<a$>", [('starttag', 'a$', [])]) self._run_check("<a$b>", [('starttag', 'a$b', [])]) self._run_check("<a$b/>", [('startendtag', 'a$b', [])]) self._run_check("<a$b >", [('starttag', 'a$b', [])]) self._run_check("<a$b />", [('startendtag', 'a$b', [])]) + self._run_check("</a$b>", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])]) @@ -576,21 +583,50 @@ text for html, expected in data: self._run_check(html, expected) - def test_EOF_in_comments_or_decls(self): + def test_eof_in_comments(self): data = [ - ('<!', [('data', '<!')]), - ('<!-', [('data', '<!-')]), - ('<!--', [('data', '<!--')]), - ('<![', [('data', '<![')]), - ('<![CDATA[', [('data', '<![CDATA[')]), - ('<![CDATA[x', [('data', '<![CDATA[x')]), - ('<!DOCTYPE', [('data', '<!DOCTYPE')]), - ('<!DOCTYPE HTML', [('data', '<!DOCTYPE HTML')]), + ('<!--', [('comment', '')]), + ('<!---', [('comment', '')]), + ('<!----', [('comment', '')]), + ('<!-----', [('comment', '-')]), + ('<!------', [('comment', '--')]), + ('<!----!', [('comment', '')]), + ('<!---!', [('comment', '-!')]), + ('<!---!>', [('comment', '-!>')]), + ('<!--foo', [('comment', 'foo')]), + ('<!--foo-', [('comment', 'foo')]), + ('<!--foo--', [('comment', 'foo')]), + ('<!--foo--!', [('comment', 'foo')]), + ('<!--<!--', [('comment', '<!')]), + ('<!--<!--!', [('comment', '<!')]), ] for html, expected in data: self._run_check(html, expected) + + def test_eof_in_declarations(self): + data = [ + ('<!', [('comment', '')]), + ('<!-', [('comment', '-')]), + ('<![', [('comment', '[')]), + ('<![CDATA[', [('unknown decl', 'CDATA[')]), + ('<![CDATA[x', [('unknown decl', 'CDATA[x')]), + ('<![CDATA[x]', [('unknown decl', 'CDATA[x]')]), + ('<![CDATA[x]]', [('unknown decl', 'CDATA[x]]')]), + ('<!DOCTYPE', [('decl', 'DOCTYPE')]), + ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]), + ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]), + ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]), + ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]), + ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]), + ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo', + [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]), + ] + for html, expected in data: + self._run_check(html, expected) + def test_bogus_comments(self): - html = ('<! not really a comment >' + html = ('<!ELEMENT br EMPTY>' + '<! not really a comment >' '<! not a comment either -->' '<! -- close enough -->' '<!><!<-- this was an empty comment>' @@ -604,6 +640,7 @@ text '<![CDATA]]>' # required '[' after CDATA ) expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), @@ -684,6 +721,26 @@ text ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check("<a " * n) + check("<a a=" * n) + check("</a " * 14 * n) + check("</a a=" * 11 * n) + check("<!--" * 4 * n) + check("<!" * 60 * n) + check("<?" * 19 * n) + check("</$" * 15 * n) + check("<![CDATA[" * 9 * n) + check("<!doctype" * 35 * n) + class AttributesTestCase(TestCaseBase): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index e584fb417b9..79eb103224b 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5875,9 +5875,9 @@ class TestSignatureDefinitions(unittest.TestCase): self._test_module_has_signatures(operator) def test_os_module_has_signatures(self): - unsupported_signature = {'chmod', 'link', 'utime'} + unsupported_signature = {'chmod', 'utime'} unsupported_signature |= {name for name in - ['get_terminal_size', 'posix_spawn', 'posix_spawnp', + ['get_terminal_size', 'link', 'posix_spawn', 'posix_spawnp', 'register_at_fork', 'startfile'] if hasattr(os, name)} self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index b3c9ef8efba..0ee4582b5d1 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -13,11 +13,11 @@ from test.support import script_helper from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') +from concurrent import interpreters from test.support import Py_GIL_DISABLED -from test.support import interpreters from test.support import force_not_colorized import test._crossinterp_definitions as defs -from test.support.interpreters import ( +from concurrent.interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) from .utils import ( @@ -133,7 +133,7 @@ class CreateTests(TestBase): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() print(interp.id) """)) @@ -196,7 +196,7 @@ class GetCurrentTests(TestBase): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(cur.id) """)) @@ -213,7 +213,7 @@ class GetCurrentTests(TestBase): with self.subTest('subinterpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) cur = interpreters.get_current() @@ -225,7 +225,7 @@ class GetCurrentTests(TestBase): with self.subTest('per-interpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) """)) @@ -582,7 +582,7 @@ class TestInterpreterClose(TestBase): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.Interpreter({interp.id}) try: interp.close() @@ -599,7 +599,7 @@ class TestInterpreterClose(TestBase): self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) interp1.exec(dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() interp3 = interpreters.create() @@ -806,7 +806,7 @@ class TestInterpreterExec(TestBase): ham() """) scriptfile = self.make_script('script.py', tempdir, text=""" - from test.support import interpreters + from concurrent import interpreters def script(): import spam @@ -827,7 +827,7 @@ class TestInterpreterExec(TestBase): ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} raise ExecutionFailed(excinfo) - test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! + concurrent.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -944,6 +944,22 @@ class TestInterpreterExec(TestBase): with self.assertRaisesRegex(InterpreterError, 'unrecognized'): interp.exec('raise Exception("it worked!")') + def test_list_comprehension(self): + # gh-135450: List comprehensions caused an assertion failure + # in _PyCode_CheckNoExternalState() + import string + r_interp, w_interp = self.pipe() + + interp = interpreters.create() + interp.exec(f"""if True: + import os + comp = [str(i) for i in range(10)] + os.write({w_interp}, ''.join(comp).encode()) + """) + self.assertEqual(os.read(r_interp, 10).decode(), string.digits) + interp.close() + + # test__interpreters covers the remaining # Interpreter.exec() behavior. @@ -1281,7 +1297,7 @@ class TestInterpreterCall(TestBase): # no module indirection with self.subTest('no indirection'): text = run(f""" - from test.support import interpreters + from concurrent import interpreters def spam(): # This a global var... @@ -1301,7 +1317,7 @@ class TestInterpreterCall(TestBase): """) with self.subTest('indirect as func, direct interp'): text = run(f""" - from test.support import interpreters + from concurrent import interpreters import mymod def spam(): @@ -1317,7 +1333,7 @@ class TestInterpreterCall(TestBase): # indirect as func, indirect interp new_mod('mymod', f""" - from test.support import interpreters + from concurrent import interpreters def run(func): interp = interpreters.create() return interp.call(func) @@ -1356,6 +1372,187 @@ class TestInterpreterCall(TestBase): with self.assertRaises(interpreters.NotShareableError): interp.call(defs.spam_returns_arg, arg) + def test_func_in___main___hidden(self): + # When a top-level function that uses global variables is called + # through Interpreter.call(), it will be pickled, sent over, + # and unpickled. That requires that it be found in the other + # interpreter's __main__ module. However, the original script + # that defined the function is only run in the main interpreter, + # so pickle.loads() would normally fail. + # + # We work around this by running the script in the other + # interpreter. However, this is a one-off solution for the sake + # of unpickling, so we avoid modifying that interpreter's + # __main__ module by running the script in a hidden module. + # + # In this test we verify that the function runs with the hidden + # module as its __globals__ when called in the other interpreter, + # and that the interpreter's __main__ module is unaffected. + text = dedent(""" + eggs = True + + def spam(*, explicit=False): + if explicit: + import __main__ + ns = __main__.__dict__ + else: + # For now we have to have a LOAD_GLOBAL in the + # function in order for globals() to actually return + # spam.__globals__. Maybe it doesn't go through pickle? + # XXX We will fix this later. + spam + ns = globals() + + func = ns.get('spam') + return [ + id(ns), + ns.get('__name__'), + ns.get('__file__'), + id(func), + None if func is None else repr(func), + ns.get('eggs'), + ns.get('ham'), + ] + + if __name__ == "__main__": + from concurrent import interpreters + interp = interpreters.create() + + ham = True + print([ + [ + spam(explicit=True), + spam(), + ], + [ + interp.call(spam, explicit=True), + interp.call(spam), + ], + ]) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + local, remote = eval(stdout) + + # In the main interpreter. + main, unpickled = local + nsid, _, _, funcid, func, _, _ = main + self.assertEqual(main, [ + nsid, + '__main__', + filename, + funcid, + func, + True, + True, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^<function spam at 0x.*>$') + self.assertEqual(unpickled, main) + + # In the subinterpreter. + main, unpickled = remote + nsid1, _, _, funcid1, _, _, _ = main + self.assertEqual(main, [ + nsid1, + '__main__', + None, + funcid1, + None, + None, + None, + ]) + nsid2, _, _, funcid2, func, _, _ = unpickled + self.assertEqual(unpickled, [ + nsid2, + '<fake __main__>', + filename, + funcid2, + func, + True, + None, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^<function spam at 0x.*>$') + self.assertNotEqual(nsid2, nsid1) + self.assertNotEqual(funcid2, funcid1) + + def test_func_in___main___uses_globals(self): + # See the note in test_func_in___main___hidden about pickle + # and the __main__ module. + # + # Additionally, the solution to that problem must provide + # for global variables on which a pickled function might rely. + # + # To check that, we run a script that has two global functions + # and a global variable in the __main__ module. One of the + # functions sets the global variable and the other returns + # the value. + # + # The script calls those functions multiple times in another + # interpreter, to verify the following: + # + # * the global variable is properly initialized + # * the global variable retains state between calls + # * the setter modifies that persistent variable + # * the getter uses the variable + # * the calls in the other interpreter do not modify + # the main interpreter + # * those calls don't modify the interpreter's __main__ module + # * the functions and variable do not actually show up in the + # other interpreter's __main__ module + text = dedent(""" + count = 0 + + def inc(x=1): + global count + count += x + + def get_count(): + return count + + if __name__ == "__main__": + counts = [] + results = [count, counts] + + from concurrent import interpreters + interp = interpreters.create() + + val = interp.call(get_count) + counts.append(val) + + interp.call(inc) + val = interp.call(get_count) + counts.append(val) + + interp.call(inc, 3) + val = interp.call(get_count) + counts.append(val) + + results.append(count) + + modified = {name: interp.call(eval, f'{name!r} in vars()') + for name in ('count', 'inc', 'get_count')} + results.append(modified) + + print(results) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + before, counts, after, modified = eval(stdout) + self.assertEqual(modified, { + 'count': False, + 'inc': False, + 'get_count': False, + }) + self.assertEqual(before, 0) + self.assertEqual(after, 0) + self.assertEqual(counts, [0, 1, 4]) + def test_raises(self): interp = interpreters.create() with self.assertRaises(ExecutionFailed): @@ -1414,6 +1611,113 @@ class TestInterpreterCall(TestBase): with self.assertRaises(interpreters.NotShareableError): interp.call(func, op, 'eggs!') + def test_callable_requires_frame(self): + # There are various functions that require a current frame. + interp = interpreters.create() + for call, expected in [ + ((eval, '[1, 2, 3]'), + [1, 2, 3]), + ((eval, 'sum([1, 2, 3])'), + 6), + ((exec, '...'), + None), + ]: + with self.subTest(str(call)): + res = interp.call(*call) + self.assertEqual(res, expected) + + result_not_pickleable = [ + globals, + locals, + vars, + ] + for func, expectedtype in { + globals: dict, + locals: dict, + vars: dict, + dir: list, + }.items(): + with self.subTest(str(func)): + if func in result_not_pickleable: + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + else: + res = interp.call(func) + self.assertIsInstance(res, expectedtype) + self.assertIn('__builtins__', res) + + def test_globals_from_builtins(self): + # The builtins exec(), eval(), globals(), locals(), vars(), + # and dir() each runs relative to the target interpreter's + # __main__ module, when called directly. However, + # globals(), locals(), and vars() don't work when called + # directly so we don't check them. + from _frozen_importlib import BuiltinImporter + interp = interpreters.create() + + names = interp.call(dir) + self.assertEqual(names, [ + '__builtins__', + '__doc__', + '__loader__', + '__name__', + '__package__', + '__spec__', + ]) + + values = {name: interp.call(eval, name) + for name in names if name != '__builtins__'} + self.assertEqual(values, { + '__name__': '__main__', + '__doc__': None, + '__spec__': None, # It wasn't imported, so no module spec? + '__package__': None, + '__loader__': BuiltinImporter, + }) + with self.assertRaises(ExecutionFailed): + interp.call(eval, 'spam'), + + interp.call(exec, f'assert dir() == {names}') + + # Update the interpreter's __main__. + interp.prepare_main(spam=42) + expected = names + ['spam'] + + names = interp.call(dir) + self.assertEqual(names, expected) + + value = interp.call(eval, 'spam') + self.assertEqual(value, 42) + + interp.call(exec, f'assert dir() == {expected}, dir()') + + def test_globals_from_stateless_func(self): + # A stateless func, which doesn't depend on any globals, + # doesn't go through pickle, so it runs in __main__. + def set_global(name, value): + globals()[name] = value + + def get_global(name): + return globals().get(name) + + interp = interpreters.create() + + modname = interp.call(get_global, '__name__') + self.assertEqual(modname, '__main__') + + res = interp.call(get_global, 'spam') + self.assertIsNone(res) + + interp.exec('spam = True') + res = interp.call(get_global, 'spam') + self.assertTrue(res) + + interp.call(set_global, 'spam', 42) + res = interp.call(get_global, 'spam') + self.assertEqual(res, 42) + + interp.exec('assert spam == 42, repr(spam)') + def test_call_in_thread(self): interp = interpreters.create() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 0c027b17cea..109ddf34453 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -8,8 +8,8 @@ import time from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_interpchannels') -from test.support import interpreters -from test.support.interpreters import channels +from concurrent import interpreters +from test.support import channels from .utils import _run_output, TestBase @@ -171,7 +171,7 @@ class TestSendRecv(TestBase): def test_send_recv_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -244,7 +244,7 @@ class TestSendRecv(TestBase): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -387,7 +387,7 @@ class TestSendRecv(TestBase): interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) obj1 = b'spam' obj2 = b'eggs' @@ -482,7 +482,7 @@ class TestSendRecv(TestBase): self.assertEqual(_channels.get_count(rch.id), 0) _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) sch.send_nowait(1, unbounditems=channels.UNBOUND) sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) @@ -518,7 +518,7 @@ class TestSendRecv(TestBase): sch.send_nowait(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj1 = rch.recv() @@ -526,7 +526,7 @@ class TestSendRecv(TestBase): sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj2 = rch.recv() diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index ac24f6568ac..15537ac6cc8 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -119,7 +119,7 @@ class StartupTests(TestBase): # The main interpreter's sys.path[0] should be used by subinterpreters. script = ''' import sys - from test.support import interpreters + from concurrent import interpreters orig = sys.path[0] @@ -170,7 +170,7 @@ class FinalizationTests(TestBase): # is reported, even when subinterpreters get cleaned up at the end. import subprocess argv = [sys.executable, '-c', '''if True: - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() raise Exception '''] diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 757373904d7..cb17340f581 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -7,8 +7,8 @@ import unittest from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') -from test.support import interpreters -from test.support.interpreters import queues, _crossinterp +from concurrent import interpreters +from concurrent.interpreters import _queues as queues, _crossinterp from .utils import _run_output, TestBase as _TestBase @@ -126,7 +126,7 @@ class QueueTests(TestBase): interp = interpreters.create() interp.exec(dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) """)); @@ -208,18 +208,64 @@ class TestQueueOps(TestBase): self.assertIs(after, True) def test_full(self): - expected = [False, False, False, True, False, False, False] - actual = [] - queue = queues.create(3) - for _ in range(3): - actual.append(queue.full()) - queue.put(None) - actual.append(queue.full()) - for _ in range(3): - queue.get() - actual.append(queue.full()) + for maxsize in [1, 3, 11]: + with self.subTest(f'maxsize={maxsize}'): + num_to_add = maxsize + expected = [False] * (num_to_add * 2 + 3) + expected[maxsize] = True + expected[maxsize + 1] = True + + queue = queues.create(maxsize) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) + + # no max size + for args in [(), (0,), (-1,), (-10,)]: + with self.subTest(f'maxsize={args[0]}' if args else '<default>'): + num_to_add = 13 + expected = [False] * (num_to_add * 2 + 3) + + queue = queues.create(*args) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) + + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) def test_qsize(self): expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] @@ -324,7 +370,7 @@ class TestQueueOps(TestBase): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.create() """)) for methname in ('get', 'get_nowait'): @@ -351,7 +397,7 @@ class TestQueueOps(TestBase): out = _run_output( interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' @@ -390,7 +436,7 @@ class TestQueueOps(TestBase): interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' @@ -468,7 +514,7 @@ class TestQueueOps(TestBase): queue = queues.create() interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) queue.put(1, unbounditems=queues.UNBOUND) queue.put(2, unbounditems=queues.UNBOUND_ERROR) @@ -504,14 +550,14 @@ class TestQueueOps(TestBase): queue.put(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = queue.get() queue.put(2, unbounditems=queues.UNBOUND) queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj2 = queue.get() obj1 = queue.get() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index fae2f38cb55..e25e67a0d4f 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -6,7 +6,7 @@ from test.support import import_helper from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. import_helper.import_module('_interpreters') -from test.support import interpreters +from concurrent import interpreters from .utils import TestBase diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index c25e0fb7475..ae09aa457b4 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -21,7 +21,7 @@ try: import _interpreters except ImportError as exc: raise unittest.SkipTest(str(exc)) -from test.support import interpreters +from concurrent import interpreters try: diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 9ea2679c77e..30f9bb33316 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -270,7 +270,7 @@ class TestMain(unittest.TestCase): (r'" \"foo\" "', f'{t.string}" \\"foo\\" "{t.reset}'), ('"α"', f'{t.string}"\\u03b1"{t.reset}'), ('123', f'{t.number}123{t.reset}'), - ('-1.2345e+23', f'{t.number}-1.2345e+23{t.reset}'), + ('-1.25e+23', f'{t.number}-1.25e+23{t.reset}'), (r'{"\\": ""}', f'''\ {ob} diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 455d2af37ef..55b502e52ca 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -387,6 +387,10 @@ class NormalizeTest(unittest.TestCase): self.check('c', 'C') self.check('posix', 'C') + def test_c_utf8(self): + self.check('c.utf8', 'C.UTF-8') + self.check('C.UTF-8', 'C.UTF-8') + def test_english(self): self.check('en', 'en_US.ISO8859-1') self.check('EN', 'en_US.ISO8859-1') diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index fa5b1e43816..3819965ed2c 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1036,7 +1036,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True - allow_reuse_port = True + allow_reuse_port = False def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): @@ -4214,89 +4214,6 @@ class ConfigDictTest(BaseTest): handler = logging.getHandlerByName('custom') self.assertEqual(handler.custom_kwargs, custom_kwargs) - # See gh-91555 and gh-90321 - @support.requires_subprocess() - def test_deadlock_in_queue(self): - queue = multiprocessing.Queue() - handler = logging.handlers.QueueHandler(queue) - logger = multiprocessing.get_logger() - level = logger.level - try: - logger.setLevel(logging.DEBUG) - logger.addHandler(handler) - logger.debug("deadlock") - finally: - logger.setLevel(level) - logger.removeHandler(handler) - - def test_recursion_in_custom_handler(self): - class BadHandler(logging.Handler): - def __init__(self): - super().__init__() - def emit(self, record): - logger.debug("recurse") - logger = logging.getLogger("test_recursion_in_custom_handler") - logger.addHandler(BadHandler()) - logger.setLevel(logging.DEBUG) - logger.debug("boom") - - @threading_helper.requires_working_threading() - def test_thread_supression_noninterference(self): - lock = threading.Lock() - logger = logging.getLogger("test_thread_supression_noninterference") - - # Block on the first call, allow others through - # - # NOTE: We need to bypass the base class's lock, otherwise that will - # block multiple calls to the same handler itself. - class BlockOnceHandler(TestHandler): - def __init__(self, barrier): - super().__init__(support.Matcher()) - self.barrier = barrier - - def createLock(self): - self.lock = None - - def handle(self, record): - self.emit(record) - - def emit(self, record): - if self.barrier: - barrier = self.barrier - self.barrier = None - barrier.wait() - with lock: - pass - super().emit(record) - logger.info("blow up if not supressed") - - barrier = threading.Barrier(2) - handler = BlockOnceHandler(barrier) - logger.addHandler(handler) - logger.setLevel(logging.DEBUG) - - t1 = threading.Thread(target=logger.debug, args=("1",)) - with lock: - - # Ensure first thread is blocked in the handler, hence supressing logging... - t1.start() - barrier.wait() - - # ...but the second thread should still be able to log... - t2 = threading.Thread(target=logger.debug, args=("2",)) - t2.start() - t2.join(timeout=3) - - self.assertEqual(len(handler.buffer), 1) - self.assertTrue(handler.matches(levelno=logging.DEBUG, message='2')) - - # The first thread should still be blocked here - self.assertTrue(t1.is_alive()) - - # Now the lock has been released the first thread should complete - t1.join() - self.assertEqual(len(handler.buffer), 2) - self.assertTrue(handler.matches(levelno=logging.DEBUG, message='1')) class ManagerTest(BaseTest): def test_manager_loggerclass(self): diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 81e11a293cc..9d720f62710 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,11 +1,7 @@ import netrc, os, unittest, sys, textwrap +from test import support from test.support import os_helper -try: - import pwd -except ImportError: - pwd = None - temp_filename = os_helper.TESTFN class NetrcTestCase(unittest.TestCase): @@ -269,9 +265,14 @@ class NetrcTestCase(unittest.TestCase): machine bar.domain.com login foo password pass """, '#pass') + @unittest.skipUnless(support.is_wasi, 'WASI only test') + def test_security_on_WASI(self): + self.assertFalse(netrc._can_security_check()) + self.assertEqual(netrc._getpwuid(0), 'uid 0') + self.assertEqual(netrc._getpwuid(123456), 'uid 123456') @unittest.skipUnless(os.name == 'posix', 'POSIX only test') - @unittest.skipIf(pwd is None, 'security check requires pwd module') + @unittest.skipUnless(hasattr(os, 'getuid'), "os.getuid is required") @os_helper.skip_unless_working_chmod def test_security(self): # This test is incomplete since we are normally not run as root and diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index e6ffd2b0ffe..e476e472780 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -615,9 +615,9 @@ Options: self.parser.add_option( "-p", "--prob", help="blow up with probability PROB [default: %default]") - self.parser.set_defaults(prob=0.43) + self.parser.set_defaults(prob=0.25) expected_help = self.help_prefix + \ - " -p PROB, --prob=PROB blow up with probability PROB [default: 0.43]\n" + " -p PROB, --prob=PROB blow up with probability PROB [default: 0.25]\n" self.assertHelp(self.parser, expected_help) def test_alt_expand(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 88b5b0e6e35..5217037ae9d 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,7 +13,6 @@ import itertools import locale import os import pickle -import platform import select import selectors import shutil @@ -1919,6 +1918,10 @@ class MakedirTests(unittest.TestCase): support.is_wasi, "WASI's umask is a stub." ) + @unittest.skipIf( + support.is_emscripten, + "TODO: Fails in buildbot; see #135783" + ) def test_mode(self): with os_helper.temp_umask(0o002): base = os_helper.TESTFN @@ -4291,13 +4294,8 @@ class EventfdTests(unittest.TestCase): @unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # 1 ms accuracy is reliably achievable on every platform except Android - # emulators, where we allow 10 ms (gh-108277). - if sys.platform == "android" and platform.android_ver().is_emulator: - CLOCK_RES_PLACES = 2 - else: - CLOCK_RES_PLACES = 3 - + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms CLOCK_RES = 10 ** -CLOCK_RES_PLACES CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 13356b4cfe0..b2e2cdb3338 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -2954,7 +2954,13 @@ class PathTest(PurePathTest): else: # ".." segments are normalized first on Windows, so this path is stat()able. self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + if sys.platform == "emscripten": + # Emscripten will return ELOOP if there are 49 or more ..'s. + # Can remove when https://github.com/emscripten-core/emscripten/pull/24591 is merged. + NDOTDOTS = 48 + else: + NDOTDOTS = 50 + self.assertEqual(set(p.glob("/".join([".."] * NDOTDOTS))), { P(self.base, *[".."] * NDOTDOTS)}) def test_glob_inaccessible(self): P = self.cls diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index ef596630b93..3d7300e1480 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -718,9 +718,9 @@ class TestTranforms(BytecodeTestCase): self.assertEqual(format('x = %d!', 1234), 'x = 1234!') self.assertEqual(format('x = %x!', 1234), 'x = 4d2!') self.assertEqual(format('x = %f!', 1234), 'x = 1234.000000!') - self.assertEqual(format('x = %s!', 1234.5678901), 'x = 1234.5678901!') - self.assertEqual(format('x = %f!', 1234.5678901), 'x = 1234.567890!') - self.assertEqual(format('x = %d!', 1234.5678901), 'x = 1234!') + self.assertEqual(format('x = %s!', 1234.0000625), 'x = 1234.0000625!') + self.assertEqual(format('x = %f!', 1234.0000625), 'x = 1234.000063!') + self.assertEqual(format('x = %d!', 1234.0000625), 'x = 1234!') self.assertEqual(format('x = %s%% %%%%', 1234), 'x = 1234% %%') self.assertEqual(format('x = %s!', '%% %s'), 'x = %% %s!') self.assertEqual(format('x = %s, y = %d', 12, 34), 'x = 12, y = 34') diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 21d097dbb55..7529c853f9c 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -506,9 +506,12 @@ def _is_perf_version_at_least(major, minor): # The output of perf --version looks like "perf version 6.7-3" but # it can also be perf version "perf version 5.15.143", or even include # a commit hash in the version string, like "6.12.9.g242e6068fd5c" + # + # PermissionError is raised if perf does not exist on the Windows Subsystem + # for Linux, see #134987 try: output = subprocess.check_output(["perf", "--version"], text=True) - except (subprocess.CalledProcessError, FileNotFoundError): + except (subprocess.CalledProcessError, FileNotFoundError, PermissionError): return False version = output.split()[2] version = version.split("-")[0] diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 0c84d3d3bfd..41c337ade7e 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -458,7 +458,7 @@ class QueryTestCase(unittest.TestCase): return super().__new__(Temperature, celsius_degrees) def __repr__(self): kelvin_degrees = self + 273.15 - return f"{kelvin_degrees}°K" + return f"{kelvin_degrees:.2f}°K" self.assertEqual(pprint.pformat(Temperature(1000)), '1273.15°K') def test_sorted_dict(self): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 54910cd8054..0217ebd132b 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -14,6 +14,15 @@ from test import support from fractions import Fraction from collections import abc, Counter + +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + class TestBasicOps: # Superclass with tests common to all generators. # Subclasses must arrange for self.gen to retrieve the Random instance @@ -142,6 +151,7 @@ class TestBasicOps: # Exception raised if size of sample exceeds that of population self.assertRaises(ValueError, self.gen.sample, population, N+1) self.assertRaises(ValueError, self.gen.sample, [], -1) + self.assertRaises(TypeError, self.gen.sample, population, 1.0) def test_sample_distribution(self): # For the entire allowable range of 0 <= k <= N, validate that @@ -259,6 +269,7 @@ class TestBasicOps: choices(data, range(4), k=5), choices(k=5, population=data, weights=range(4)), choices(k=5, population=data, cum_weights=range(4)), + choices(data, k=MyIndex(5)), ]: self.assertEqual(len(sample), 5) self.assertEqual(type(sample), list) @@ -369,122 +380,40 @@ class TestBasicOps: self.assertEqual(x1, x2) self.assertEqual(y1, y2) + @support.requires_IEEE_754 + def test_53_bits_per_float(self): + span = 2 ** 53 + cum = 0 + for i in range(100): + cum |= int(self.gen.random() * span) + self.assertEqual(cum, span-1) + def test_getrandbits(self): + getrandbits = self.gen.getrandbits # Verify ranges for k in range(1, 1000): - self.assertTrue(0 <= self.gen.getrandbits(k) < 2**k) - self.assertEqual(self.gen.getrandbits(0), 0) + self.assertTrue(0 <= getrandbits(k) < 2**k) + self.assertEqual(getrandbits(0), 0) # Verify all bits active - getbits = self.gen.getrandbits for span in [1, 2, 3, 4, 31, 32, 32, 52, 53, 54, 119, 127, 128, 129]: all_bits = 2**span-1 cum = 0 cpl_cum = 0 for i in range(100): - v = getbits(span) + v = getrandbits(span) cum |= v cpl_cum |= all_bits ^ v self.assertEqual(cum, all_bits) self.assertEqual(cpl_cum, all_bits) # Verify argument checking - self.assertRaises(TypeError, self.gen.getrandbits) - self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) - self.assertRaises(ValueError, self.gen.getrandbits, -1) - self.assertRaises(OverflowError, self.gen.getrandbits, 1<<1000) - self.assertRaises(ValueError, self.gen.getrandbits, -1<<1000) - self.assertRaises(TypeError, self.gen.getrandbits, 10.1) - - def test_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - state = pickle.dumps(self.gen, proto) - origseq = [self.gen.random() for i in range(10)] - newgen = pickle.loads(state) - restoredseq = [newgen.random() for i in range(10)] - self.assertEqual(origseq, restoredseq) - - def test_bug_1727780(self): - # verify that version-2-pickles can be loaded - # fine, whether they are created on 32-bit or 64-bit - # platforms, and that version-3-pickles load fine. - files = [("randv2_32.pck", 780), - ("randv2_64.pck", 866), - ("randv3.pck", 343)] - for file, value in files: - with open(support.findfile(file),"rb") as f: - r = pickle.load(f) - self.assertEqual(int(r.random()*1000), value) - - def test_bug_9025(self): - # Had problem with an uneven distribution in int(n*random()) - # Verify the fix by checking that distributions fall within expectations. - n = 100000 - randrange = self.gen.randrange - k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) - self.assertTrue(0.30 < k/n < .37, (k/n)) - - def test_randbytes(self): - # Verify ranges - for n in range(1, 10): - data = self.gen.randbytes(n) - self.assertEqual(type(data), bytes) - self.assertEqual(len(data), n) - - self.assertEqual(self.gen.randbytes(0), b'') - - # Verify argument checking - self.assertRaises(TypeError, self.gen.randbytes) - self.assertRaises(TypeError, self.gen.randbytes, 1, 2) - self.assertRaises(ValueError, self.gen.randbytes, -1) - self.assertRaises(OverflowError, self.gen.randbytes, 1<<1000) - self.assertRaises((ValueError, OverflowError), self.gen.randbytes, -1<<1000) - self.assertRaises(TypeError, self.gen.randbytes, 1.0) - - def test_mu_sigma_default_args(self): - self.assertIsInstance(self.gen.normalvariate(), float) - self.assertIsInstance(self.gen.gauss(), float) - - -try: - random.SystemRandom().random() -except NotImplementedError: - SystemRandom_available = False -else: - SystemRandom_available = True - -@unittest.skipUnless(SystemRandom_available, "random.SystemRandom not available") -class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): - gen = random.SystemRandom() - - def test_autoseed(self): - # Doesn't need to do anything except not fail - self.gen.seed() - - def test_saverestore(self): - self.assertRaises(NotImplementedError, self.gen.getstate) - self.assertRaises(NotImplementedError, self.gen.setstate, None) - - def test_seedargs(self): - # Doesn't need to do anything except not fail - self.gen.seed(100) - - def test_gauss(self): - self.gen.gauss_next = None - self.gen.seed(100) - self.assertEqual(self.gen.gauss_next, None) - - def test_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertRaises(NotImplementedError, pickle.dumps, self.gen, proto) - - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) + self.assertRaises(TypeError, getrandbits) + self.assertRaises(TypeError, getrandbits, 1, 2) + self.assertRaises(ValueError, getrandbits, -1) + self.assertRaises(OverflowError, getrandbits, 1<<1000) + self.assertRaises(ValueError, getrandbits, -1<<1000) + self.assertRaises(TypeError, getrandbits, 10.1) def test_bigrand(self): # The randrange routine should build-up the required number of bits @@ -563,6 +492,10 @@ class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): randrange(1000, step=100) with self.assertRaises(TypeError): randrange(1000, None, step=100) + with self.assertRaises(TypeError): + randrange(1000, step=MyIndex(1)) + with self.assertRaises(TypeError): + randrange(1000, None, step=MyIndex(1)) def test_randbelow_logic(self, _log=log, int=int): # check bitcount transition points: 2**i and 2**(i+1)-1 @@ -585,6 +518,116 @@ class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): self.assertEqual(k, numbits) # note the stronger assertion self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion + def test_randrange_index(self): + randrange = self.gen.randrange + self.assertIn(randrange(MyIndex(5)), range(5)) + self.assertIn(randrange(MyIndex(2), MyIndex(7)), range(2, 7)) + self.assertIn(randrange(MyIndex(5), MyIndex(15), MyIndex(2)), range(5, 15, 2)) + + def test_randint(self): + randint = self.gen.randint + self.assertIn(randint(2, 5), (2, 3, 4, 5)) + self.assertEqual(randint(2, 2), 2) + self.assertIn(randint(MyIndex(2), MyIndex(5)), (2, 3, 4, 5)) + self.assertEqual(randint(MyIndex(2), MyIndex(2)), 2) + + self.assertRaises(ValueError, randint, 5, 2) + self.assertRaises(TypeError, randint) + self.assertRaises(TypeError, randint, 2) + self.assertRaises(TypeError, randint, 2, 5, 1) + self.assertRaises(TypeError, randint, 2.0, 5) + self.assertRaises(TypeError, randint, 2, 5.0) + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + state = pickle.dumps(self.gen, proto) + origseq = [self.gen.random() for i in range(10)] + newgen = pickle.loads(state) + restoredseq = [newgen.random() for i in range(10)] + self.assertEqual(origseq, restoredseq) + + def test_bug_1727780(self): + # verify that version-2-pickles can be loaded + # fine, whether they are created on 32-bit or 64-bit + # platforms, and that version-3-pickles load fine. + files = [("randv2_32.pck", 780), + ("randv2_64.pck", 866), + ("randv3.pck", 343)] + for file, value in files: + with open(support.findfile(file),"rb") as f: + r = pickle.load(f) + self.assertEqual(int(r.random()*1000), value) + + def test_bug_9025(self): + # Had problem with an uneven distribution in int(n*random()) + # Verify the fix by checking that distributions fall within expectations. + n = 100000 + randrange = self.gen.randrange + k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) + self.assertTrue(0.30 < k/n < .37, (k/n)) + + def test_randrange_bug_1590891(self): + start = 1000000000000 + stop = -100000000000000000000 + step = -200 + x = self.gen.randrange(start, stop, step) + self.assertTrue(stop < x <= start) + self.assertEqual((x+stop)%step, 0) + + def test_randbytes(self): + # Verify ranges + for n in range(1, 10): + data = self.gen.randbytes(n) + self.assertEqual(type(data), bytes) + self.assertEqual(len(data), n) + + self.assertEqual(self.gen.randbytes(0), b'') + + # Verify argument checking + self.assertRaises(TypeError, self.gen.randbytes) + self.assertRaises(TypeError, self.gen.randbytes, 1, 2) + self.assertRaises(ValueError, self.gen.randbytes, -1) + self.assertRaises(OverflowError, self.gen.randbytes, 1<<1000) + self.assertRaises((ValueError, OverflowError), self.gen.randbytes, -1<<1000) + self.assertRaises(TypeError, self.gen.randbytes, 1.0) + + def test_mu_sigma_default_args(self): + self.assertIsInstance(self.gen.normalvariate(), float) + self.assertIsInstance(self.gen.gauss(), float) + + +try: + random.SystemRandom().random() +except NotImplementedError: + SystemRandom_available = False +else: + SystemRandom_available = True + +@unittest.skipUnless(SystemRandom_available, "random.SystemRandom not available") +class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): + gen = random.SystemRandom() + + def test_autoseed(self): + # Doesn't need to do anything except not fail + self.gen.seed() + + def test_saverestore(self): + self.assertRaises(NotImplementedError, self.gen.getstate) + self.assertRaises(NotImplementedError, self.gen.setstate, None) + + def test_seedargs(self): + # Doesn't need to do anything except not fail + self.gen.seed(100) + + def test_gauss(self): + self.gen.gauss_next = None + self.gen.seed(100) + self.assertEqual(self.gen.gauss_next, None) + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(NotImplementedError, pickle.dumps, self.gen, proto) + class TestRawMersenneTwister(unittest.TestCase): @test.support.cpython_only @@ -770,38 +813,6 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): seed = (1 << (10000 * 8)) - 1 # about 10K bytes self.gen.seed(seed) - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) - - def test_bigrand(self): - # The randrange routine should build-up the required number of bits - # in stages so that all bit positions are active. - span = 2 ** 500 - cum = 0 - for i in range(100): - r = self.gen.randrange(span) - self.assertTrue(0 <= r < span) - cum |= r - self.assertEqual(cum, span-1) - - def test_bigrand_ranges(self): - for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: - start = self.gen.randrange(2 ** (i-2)) - stop = self.gen.randrange(2 ** i) - if stop <= start: - continue - self.assertTrue(start <= self.gen.randrange(start, stop) < stop) - - def test_rangelimits(self): - for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: - self.assertEqual(set(range(start,stop)), - set([self.gen.randrange(start,stop) for i in range(100)])) - def test_getrandbits(self): super().test_getrandbits() @@ -809,6 +820,9 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.gen.seed(1234567) self.assertEqual(self.gen.getrandbits(100), 97904845777343510404718956115) + self.gen.seed(1234567) + self.assertEqual(self.gen.getrandbits(MyIndex(100)), + 97904845777343510404718956115) def test_getrandbits_2G_bits(self): size = 2**31 @@ -836,27 +850,6 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.assertEqual(self.gen.randrange(2**99), 97904845777343510404718956115) - def test_randbelow_logic(self, _log=log, int=int): - # check bitcount transition points: 2**i and 2**(i+1)-1 - # show that: k = int(1.001 + _log(n, 2)) - # is equal to or one greater than the number of bits in n - for i in range(1, 1000): - n = 1 << i # check an exact power of two - numbits = i+1 - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) - self.assertEqual(n, 2**(k-1)) - - n += n - 1 # check 1 below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertIn(k, [numbits, numbits+1]) - self.assertTrue(2**k > n > 2**(k-2)) - - n -= n >> 15 # check a little farther below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) # note the stronger assertion - self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion - def test_randbelow_without_getrandbits(self): # Random._randbelow() can only use random() when the built-in one # has been overridden but no new getrandbits() method was supplied. @@ -891,14 +884,6 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase): self.gen._randbelow_without_getrandbits(n, maxsize=maxsize) self.assertEqual(random_mock.call_count, 2) - def test_randrange_bug_1590891(self): - start = 1000000000000 - stop = -100000000000000000000 - step = -200 - x = self.gen.randrange(start, stop, step) - self.assertTrue(stop < x <= start) - self.assertEqual((x+stop)%step, 0) - def test_choices_algorithms(self): # The various ways of specifying weights should produce the same results choices = self.gen.choices diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a43d2678ebd..5bc3c5924b0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2346,6 +2346,17 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('-j1', '-v', testname, env=env, isolated=False) check(output) + def test_pgo_exclude(self): + # Get PGO tests + output = self.run_tests('--pgo', '--list-tests') + pgo_tests = output.strip().split() + + # Exclude test_re + output = self.run_tests('--pgo', '--list-tests', '-x', 'test_re') + tests = output.strip().split() + self.assertNotIn('test_re', tests) + self.assertEqual(len(tests), len(pgo_tests) - 1) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 16623654c29..22a55b57c07 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -151,14 +151,38 @@ class ReprTests(unittest.TestCase): eq(r(frozenset({1, 2, 3, 4, 5, 6, 7})), "frozenset({1, 2, 3, 4, 5, 6, ...})") def test_numbers(self): - eq = self.assertEqual - eq(r(123), repr(123)) - eq(r(123), repr(123)) - eq(r(1.0/3), repr(1.0/3)) - - n = 10**100 - expected = repr(n)[:18] + "..." + repr(n)[-19:] - eq(r(n), expected) + for x in [123, 1.0 / 3]: + self.assertEqual(r(x), repr(x)) + + max_digits = sys.get_int_max_str_digits() + for k in [100, max_digits - 1]: + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + expected = repr(n)[:18] + "..." + repr(n)[-19:] + self.assertEqual(r(n), expected) + + def re_msg(n, d): + return (rf'<{n.__class__.__name__} instance with roughly {d} ' + rf'digits \(limit at {max_digits}\) at 0x[a-f0-9]+>') + + k = max_digits + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + + for k in [max_digits + 1, 2 * max_digits]: + self.assertGreater(k, 100) + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + with self.subTest(f'10 ** {k} - 1', k=k): + n = 10 ** k - 1 + # Here, since math.log10(n) == math.log10(n-1), + # the number of digits of n - 1 is overestimated. + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) def test_instance(self): eq = self.assertEqual @@ -373,20 +397,20 @@ class ReprTests(unittest.TestCase): 'object': { 1: 'two', b'three': [ - (4.5, 6.7), + (4.5, 6.25), [set((8, 9)), frozenset((10, 11))], ], }, 'tests': ( (dict(indent=None), '''\ - {1: 'two', b'three': [(4.5, 6.7), [{8, 9}, frozenset({10, 11})]]}'''), + {1: 'two', b'three': [(4.5, 6.25), [{8, 9}, frozenset({10, 11})]]}'''), (dict(indent=False), '''\ { 1: 'two', b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -406,7 +430,7 @@ class ReprTests(unittest.TestCase): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -426,7 +450,7 @@ class ReprTests(unittest.TestCase): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -446,7 +470,7 @@ class ReprTests(unittest.TestCase): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -466,7 +490,7 @@ class ReprTests(unittest.TestCase): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -494,7 +518,7 @@ class ReprTests(unittest.TestCase): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -514,7 +538,7 @@ class ReprTests(unittest.TestCase): -->b'three': [ -->-->( -->-->-->4.5, - -->-->-->6.7, + -->-->-->6.25, -->-->), -->-->[ -->-->-->{ @@ -534,7 +558,7 @@ class ReprTests(unittest.TestCase): ....b'three': [ ........( ............4.5, - ............6.7, + ............6.25, ........), ........[ ............{ diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 37e0f74f688..720fa3c4c1e 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -1,14 +1,22 @@ """sqlite3 CLI tests.""" import sqlite3 +import sys +import textwrap import unittest +import unittest.mock +import os from sqlite3.__main__ import main as cli +from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink +from test.support.pty_helper import run_pty from test.support import ( captured_stdout, captured_stderr, captured_stdin, force_not_colorized_test_class, + requires_subprocess, + verbose, ) @@ -130,7 +138,7 @@ class InteractiveSession(unittest.TestCase): self.assertEndsWith(out, self.PS1) self.assertEqual(out.count(self.PS1), 2) self.assertEqual(out.count(self.PS2), 0) - self.assertIn("Error", err) + self.assertIn('Error: unknown command: "', err) # test "unknown_command" is pointed out in the error message self.assertIn("unknown_command", err) @@ -200,5 +208,108 @@ class InteractiveSession(unittest.TestCase): self.assertIn('\x1b[1;35mOperationalError (SQLITE_ERROR)\x1b[0m: ' '\x1b[35mnear "sel": syntax error\x1b[0m', err) + +@requires_subprocess() +@force_not_colorized_test_class +class Completion(unittest.TestCase): + PS1 = "sqlite> " + + @classmethod + def setUpClass(cls): + _sqlite3 = import_module("_sqlite3") + if not hasattr(_sqlite3, "SQLITE_KEYWORDS"): + raise unittest.SkipTest("unable to determine SQLite keywords") + + readline = import_module("readline") + if readline.backend == "editline": + raise unittest.SkipTest("libedit readline is not supported") + + def write_input(self, input_, env=None): + script = textwrap.dedent(""" + import readline + from sqlite3.__main__ import main + + readline.parse_and_bind("set colored-completion-prefix off") + main() + """) + return run_pty(script, input_, env) + + def test_complete_sql_keywords(self): + # List candidates starting with 'S', there should be multiple matches. + input_ = b"S\t\tEL\t 1;\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"SELECT", output) + self.assertIn(b"SET", output) + self.assertIn(b"SAVEPOINT", output) + self.assertIn(b"(1,)", output) + + # Keywords are completed in upper case for even lower case user input. + input_ = b"sel\t\t 1;\n.quit\n" + output = self.write_input(input_) + self.assertIn(b"SELECT", output) + self.assertIn(b"(1,)", output) + + @unittest.skipIf(sys.platform.startswith("freebsd"), + "Two actual tabs are inserted when there are no matching" + " completions in the pseudo-terminal opened by run_pty()" + " on FreeBSD") + def test_complete_no_match(self): + input_ = b"xyzzy\t\t\b\b\b\b\b\b\b.quit\n" + # Set NO_COLOR to disable coloring for self.PS1. + output = self.write_input(input_, env={**os.environ, "NO_COLOR": "1"}) + lines = output.decode().splitlines() + indices = ( + i for i, line in enumerate(lines, 1) + if line.startswith(f"{self.PS1}xyzzy") + ) + line_num = next(indices, -1) + self.assertNotEqual(line_num, -1) + # Completions occupy lines, assert no extra lines when there is nothing + # to complete. + self.assertEqual(line_num, len(lines)) + + def test_complete_no_input(self): + from _sqlite3 import SQLITE_KEYWORDS + + script = textwrap.dedent(""" + import readline + from sqlite3.__main__ import main + + # Configure readline to ...: + # - hide control sequences surrounding each candidate + # - hide "Display all xxx possibilities? (y or n)" + # - hide "--More--" + # - show candidates one per line + readline.parse_and_bind("set colored-completion-prefix off") + readline.parse_and_bind("set colored-stats off") + readline.parse_and_bind("set completion-query-items 0") + readline.parse_and_bind("set page-completions off") + readline.parse_and_bind("set completion-display-width 0") + readline.parse_and_bind("set show-all-if-ambiguous off") + readline.parse_and_bind("set show-all-if-unmodified off") + + main() + """) + input_ = b"\t\t.quit\n" + output = run_pty(script, input_, env={**os.environ, "NO_COLOR": "1"}) + try: + lines = output.decode().splitlines() + indices = [ + i for i, line in enumerate(lines) + if line.startswith(self.PS1) + ] + self.assertEqual(len(indices), 2) + start, end = indices + candidates = [l.strip() for l in lines[start+1:end]] + self.assertEqual(candidates, sorted(SQLITE_KEYWORDS)) + except: + if verbose: + print(' PTY output: '.center(30, '-')) + print(output.decode(errors='replace')) + print(' end PTY output '.center(30, '-')) + raise + + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 0dd619dd7c8..8250b0aef09 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -3319,7 +3319,8 @@ class TestNormalDistC(unittest.TestCase, TestNormalDist): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) - tests.addTests(doctest.DocTestSuite(statistics)) + if sys.float_repr_style == 'short': + tests.addTests(doctest.DocTestSuite(statistics)) return tests diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index d6a7bd0da59..2584fbf72d3 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -1231,10 +1231,10 @@ class StrTest(string_tests.StringLikeTest, self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00') self.assertEqual('{0:<6}'.format(3), '3 ') - self.assertEqual('{0:\x00<6}'.format(3.14), '3.14\x00\x00') - self.assertEqual('{0:\x01<6}'.format(3.14), '3.14\x01\x01') - self.assertEqual('{0:\x00^6}'.format(3.14), '\x003.14\x00') - self.assertEqual('{0:^6}'.format(3.14), ' 3.14 ') + self.assertEqual('{0:\x00<6}'.format(3.25), '3.25\x00\x00') + self.assertEqual('{0:\x01<6}'.format(3.25), '3.25\x01\x01') + self.assertEqual('{0:\x00^6}'.format(3.25), '\x003.25\x00') + self.assertEqual('{0:^6}'.format(3.25), ' 3.25 ') self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00') self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01') diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 268230f6da7..0241e543cd7 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -221,14 +221,16 @@ class StrptimeTests(unittest.TestCase): self.assertRaises(ValueError, _strptime._strptime_time, data_string="%d", format="%A") for bad_format in ("%", "% ", "%\n"): - with self.assertRaisesRegex(ValueError, "stray % in format "): + with (self.subTest(format=bad_format), + self.assertRaisesRegex(ValueError, "stray % in format ")): _strptime._strptime_time("2005", bad_format) - for bad_format in ("%e", "%Oe", "%O", "%O ", "%Ee", "%E", "%E ", - "%.", "%+", "%_", "%~", "%\\", + for bad_format in ("%i", "%Oi", "%O", "%O ", "%Ee", "%E", "%E ", + "%.", "%+", "%~", "%\\", "%O.", "%O+", "%O_", "%O~", "%O\\"): directive = bad_format[1:].rstrip() - with self.assertRaisesRegex(ValueError, - f"'{re.escape(directive)}' is a bad directive in format "): + with (self.subTest(format=bad_format), + self.assertRaisesRegex(ValueError, + f"'{re.escape(directive)}' is a bad directive in format ")): _strptime._strptime_time("2005", bad_format) msg_week_no_year_or_weekday = r"ISO week directive '%V' must be used with " \ @@ -335,6 +337,15 @@ class StrptimeTests(unittest.TestCase): self.roundtrip('%B', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) self.roundtrip('%b', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) + @run_with_locales('LC_TIME', 'az_AZ', 'ber_DZ', 'ber_MA', 'crh_UA') + def test_month_locale2(self): + # Test for month directives + # Month name contains 'İ' ('\u0130') + self.roundtrip('%B', 1, (2025, 6, 1, 0, 0, 0, 6, 152, 0)) + self.roundtrip('%b', 1, (2025, 6, 1, 0, 0, 0, 6, 152, 0)) + self.roundtrip('%B', 1, (2025, 7, 1, 0, 0, 0, 1, 182, 0)) + self.roundtrip('%b', 1, (2025, 7, 1, 0, 0, 0, 1, 182, 0)) + def test_day(self): # Test for day directives self.roundtrip('%d %Y', 2) @@ -480,13 +491,11 @@ class StrptimeTests(unittest.TestCase): # * Year is not included: ha_NG. # * Use non-Gregorian calendar: lo_LA, thai, th_TH. # On Windows: ar_IN, ar_SA, fa_IR, ps_AF. - # - # BUG: Generates regexp that does not match the current date and time - # for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG', 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', - 'my_MM', 'or_IN', 'shn_MM', 'az_IR') + 'my_MM', 'or_IN', 'shn_MM', 'az_IR', + 'byn_ER', 'wal_ET', 'lzh_TW') def test_date_time_locale(self): # Test %c directive loc = locale.getlocale(locale.LC_TIME)[0] @@ -525,11 +534,9 @@ class StrptimeTests(unittest.TestCase): # NB: Does not roundtrip because use non-Gregorian calendar: # lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF. - # BUG: Generates regexp that does not match the current date - # for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', 'eu_ES', 'ar_AE', - 'az_IR', 'my_MM', 'or_IN', 'shn_MM') + 'az_IR', 'my_MM', 'or_IN', 'shn_MM', 'lzh_TW') def test_date_locale(self): # Test %x directive now = time.time() @@ -546,7 +553,7 @@ class StrptimeTests(unittest.TestCase): # NB: Dates before 1969 do not roundtrip on many locales, including C. @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', - 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM') + 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM', 'lzh_TW') def test_date_locale2(self): # Test %x directive loc = locale.getlocale(locale.LC_TIME)[0] @@ -562,11 +569,11 @@ class StrptimeTests(unittest.TestCase): # norwegian, nynorsk. # * Hours are in 12-hour notation without AM/PM indication: hy_AM, # ms_MY, sm_WS. - # BUG: Generates regexp that does not match the current time for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET', 'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO', - 'ti_ET', 'tig_ER', 'wal_ET') + 'ti_ET', 'tig_ER', 'wal_ET', 'lzh_TW', + 'ar_SA', 'bg_BG') def test_time_locale(self): # Test %X directive loc = locale.getlocale(locale.LC_TIME)[0] diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index bf415894903..486bf10a0b5 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -24,7 +24,7 @@ from test.support import import_helper from test.support import force_not_colorized from test.support import SHORT_TIMEOUT try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None import textwrap @@ -869,12 +869,7 @@ class SysModuleTest(unittest.TestCase): def assert_raise_on_new_sys_type(self, sys_attr): # Users are intentionally prevented from creating new instances of # sys.flags, sys.version_info, and sys.getwindowsversion. - arg = sys_attr - attr_type = type(sys_attr) - with self.assertRaises(TypeError): - attr_type(arg) - with self.assertRaises(TypeError): - attr_type.__new__(attr_type, arg) + support.check_disallow_instantiation(self, type(sys_attr), sys_attr) def test_sys_flags_no_instantiation(self): self.assert_raise_on_new_sys_type(sys.flags) @@ -1074,6 +1069,7 @@ class SysModuleTest(unittest.TestCase): self.assertHasAttr(sys.implementation, 'version') self.assertHasAttr(sys.implementation, 'hexversion') self.assertHasAttr(sys.implementation, 'cache_tag') + self.assertHasAttr(sys.implementation, 'supports_isolated_interpreters') version = sys.implementation.version self.assertEqual(version[:2], (version.major, version.minor)) @@ -1087,6 +1083,15 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(sys.implementation.name, sys.implementation.name.lower()) + # https://peps.python.org/pep-0734 + sii = sys.implementation.supports_isolated_interpreters + self.assertIsInstance(sii, bool) + if test.support.check_impl_detail(cpython=True): + if test.support.is_emscripten or test.support.is_wasi: + self.assertFalse(sii) + else: + self.assertTrue(sii) + @test.support.cpython_only def test_debugmallocstats(self): # Test sys._debugmallocstats() @@ -1943,22 +1948,7 @@ class SizeofTest(unittest.TestCase): self.assertEqual(out, b"") self.assertEqual(err, b"") - -def _supports_remote_attaching(): - PROCESS_VM_READV_SUPPORTED = False - - try: - from _remote_debugging import PROCESS_VM_READV_SUPPORTED - except ImportError: - pass - - return PROCESS_VM_READV_SUPPORTED - -@unittest.skipIf(not sys.is_remote_debug_enabled(), "Remote debugging is not enabled") -@unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32", - "Test only runs on Linux, Windows and MacOS") -@unittest.skipIf(sys.platform == "linux" and not _supports_remote_attaching(), - "Test only runs on Linux with process_vm_readv support") +@test.support.support_remote_exec_only @test.support.cpython_only class TestRemoteExec(unittest.TestCase): def tearDown(self): @@ -2117,7 +2107,7 @@ print("Remote script executed successfully!") returncode, stdout, stderr = self._run_remote_exec_test(script, prologue=prologue) self.assertEqual(returncode, 0) self.assertIn(b"Remote script executed successfully!", stdout) - self.assertIn(b"Audit event: remote_debugger_script, arg: ", stdout) + self.assertIn(b"Audit event: cpython.remote_debugger_script, arg: ", stdout) self.assertEqual(stderr, b"") def test_remote_exec_with_exception(self): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 2c0df9376ab..2eb8de4b29f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -711,8 +711,8 @@ class TestSysConfig(unittest.TestCase, VirtualEnvironmentMixin): ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} for key in ignore_keys: - json_config_vars.pop(key) - system_config_vars.pop(key) + json_config_vars.pop(key, None) + system_config_vars.pop(key, None) self.assertEqual(system_config_vars, json_config_vars) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 59b3a749d2f..13b55d0f0a2 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -28,7 +28,7 @@ from test import lock_tests from test import support try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None @@ -1284,12 +1284,6 @@ class ThreadTests(BaseTestCase): @cpython_only def test_finalize_daemon_thread_hang(self): - if support.check_sanitizer(thread=True, memory=True): - # the thread running `time.sleep(100)` below will still be alive - # at process exit - self.skipTest( - "https://github.com/python/cpython/issues/124878 - Known" - " race condition that TSAN identifies.") # gh-87135: tests that daemon threads hang during finalization script = textwrap.dedent(''' import os diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index e72a1ea5417..aabae385567 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -219,6 +219,7 @@ class TestTString(unittest.TestCase, TStringBaseCase): ("t'{lambda:1}'", "t-string: lambda expressions are not allowed " "without parentheses"), ("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"), + ("t'{1:d\n}'", "t-string: newlines are not allowed in format specifiers") ): with self.subTest(case), self.assertRaisesRegex(SyntaxError, err): eval(case) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 9011e0e1962..fc26e71ffcb 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2,7 +2,7 @@ from test.support import ( run_with_locale, cpython_only, no_rerun, - MISSING_C_DOCSTRINGS, EqualToForwardRef, + MISSING_C_DOCSTRINGS, EqualToForwardRef, check_disallow_instantiation, ) from test.support.script_helper import assert_python_ok from test.support.import_helper import import_fresh_module @@ -517,8 +517,8 @@ class TypesTests(unittest.TestCase): # and a number after the decimal. This is tricky, because # a totally empty format specifier means something else. # So, just use a sign flag - test(1e200, '+g', '+1e+200') - test(1e200, '+', '+1e+200') + test(1.25e200, '+g', '+1.25e+200') + test(1.25e200, '+', '+1.25e+200') test(1.1e200, '+g', '+1.1e+200') test(1.1e200, '+', '+1.1e+200') @@ -1148,8 +1148,7 @@ class UnionTests(unittest.TestCase): msg='Check for union reference leak.') def test_instantiation(self): - with self.assertRaises(TypeError): - types.UnionType() + check_disallow_instantiation(self, types.UnionType) self.assertIs(int, types.UnionType[int]) self.assertIs(int, types.UnionType[int, int]) self.assertEqual(int | str, types.UnionType[int, str]) @@ -2513,15 +2512,16 @@ class SubinterpreterTests(unittest.TestCase): def setUpClass(cls): global interpreters try: - from test.support import interpreters + from concurrent import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') - import test.support.interpreters.channels # noqa: F401 + from test.support import channels # noqa: F401 + cls.create_channel = staticmethod(channels.create) @cpython_only @no_rerun('channels (and queues) might have a refleak; see gh-122199') def test_static_types_inherited_slots(self): - rch, sch = interpreters.channels.create() + rch, sch = self.create_channel() script = textwrap.dedent(""" import test.support @@ -2547,7 +2547,7 @@ class SubinterpreterTests(unittest.TestCase): main_results = collate_results(raw) interp = interpreters.create() - interp.exec('from test.support import interpreters') + interp.exec('from concurrent import interpreters') interp.prepare_main(sch=sch) interp.exec(script) raw = rch.recv_nowait() diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index d1e48bde982..0e82c723ec3 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1050,6 +1050,7 @@ class SpecSignatureTest(unittest.TestCase): create_autospec(WithPostInit()), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithPostInit) self.assertIsInstance(mock.a, int) self.assertIsInstance(mock.b, int) @@ -1072,6 +1073,7 @@ class SpecSignatureTest(unittest.TestCase): create_autospec(WithDefault(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithDefault) self.assertIsInstance(mock.a, int) self.assertIsInstance(mock.b, int) @@ -1087,6 +1089,7 @@ class SpecSignatureTest(unittest.TestCase): create_autospec(WithMethod(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithMethod) self.assertIsInstance(mock.a, int) mock.b.assert_not_called() @@ -1102,11 +1105,29 @@ class SpecSignatureTest(unittest.TestCase): create_autospec(WithNonFields(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithNonFields) with self.assertRaisesRegex(AttributeError, msg): mock.a with self.assertRaisesRegex(AttributeError, msg): mock.b + def test_dataclass_special_attrs(self): + @dataclass + class Description: + name: str + + for mock in [ + create_autospec(Description, instance=True), + create_autospec(Description(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock, Description) + self.assertIs(mock.__class__, Description) + self.assertIsInstance(mock.__dataclass_fields__, MagicMock) + self.assertIsInstance(mock.__dataclass_params__, MagicMock) + self.assertIsInstance(mock.__match_args__, MagicMock) + self.assertIsInstance(mock.__hash__, MagicMock) + class TestCallList(unittest.TestCase): def test_args_list_contains_call_list(self): diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index bc1030eea60..1d889ae7cf4 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1569,6 +1569,7 @@ class Pathname_Tests(unittest.TestCase): urllib.request.url2pathname, url, require_scheme=True) + @unittest.skipIf(support.is_emscripten, "Fixed by https://github.com/emscripten-core/emscripten/pull/24593") def test_url2pathname_resolve_host(self): fn = urllib.request.url2pathname sep = os.path.sep |