From bf2e7e55d7306b1e2fce7dce767e8df5ff42cf1c Mon Sep 17 00:00:00 2001 From: Asheesh Laroia Date: Sun, 7 Feb 2021 19:15:51 -0800 Subject: bpo-40692: Run more test_concurrent_futures tests (GH-20239) In the case of multiprocessing.synchronize() being missing, the test_concurrent_futures test suite now skips only the tests that require multiprocessing.synchronize(). Validate that multiprocessing.synchronize exists as part of _check_system_limits(), allowing ProcessPoolExecutor to raise NotImplementedError during __init__, rather than crashing with ImportError during __init__ when creating a lock imported from multiprocessing.synchronize. Use _check_system_limits() to disable tests of ProcessPoolExecutor on systems without multiprocessing.synchronize. Running the test suite without multiprocessing.synchronize reveals that Lib/compileall.py crashes when it uses a ProcessPoolExecutor. Therefore, change Lib/compileall.py to call _check_system_limits() before creating the ProcessPoolExecutor. Note that both Lib/compileall.py and Lib/test/test_compileall.py were attempting to sanity-check ProcessPoolExecutor by expecting ImportError. In multiprocessing.resource_tracker, sem_unlink() is also absent on platforms where POSIX semaphores aren't available. Avoid using sem_unlink() if it, too, does not exist. Co-authored-by: Pablo Galindo --- Lib/test/test_concurrent_futures.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'Lib/test/test_concurrent_futures.py') diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index a182b14fb9b..99651f5f4ed 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -4,8 +4,6 @@ from test.support import threading_helper # Skip tests if _multiprocessing wasn't built. import_helper.import_module('_multiprocessing') -# Skip tests if sem_open implementation is broken. -support.skip_if_broken_multiprocessing_synchronize() from test.support import hashlib_helper from test.support.script_helper import assert_python_ok @@ -27,7 +25,7 @@ from concurrent import futures from concurrent.futures._base import ( PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future, BrokenExecutor) -from concurrent.futures.process import BrokenProcessPool +from concurrent.futures.process import BrokenProcessPool, _check_system_limits from multiprocessing import get_context import multiprocessing.process @@ -161,6 +159,10 @@ class ProcessPoolForkMixin(ExecutorMixin): ctx = "fork" def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") return super().get_context() @@ -170,12 +172,23 @@ class ProcessPoolSpawnMixin(ExecutorMixin): executor_type = futures.ProcessPoolExecutor ctx = "spawn" + def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") + return super().get_context() + class ProcessPoolForkserverMixin(ExecutorMixin): executor_type = futures.ProcessPoolExecutor ctx = "forkserver" def get_context(self): + try: + _check_system_limits() + except NotImplementedError: + self.skipTest("ProcessPoolExecutor unavailable on this system") if sys.platform == "win32": self.skipTest("require unix system") return super().get_context() -- cgit v1.2.3