diff options
Diffstat (limited to 'Lib/importlib')
36 files changed, 1903 insertions, 718 deletions
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 2baaf937329..e62bdb1654e 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -1,108 +1,63 @@ -"""A pure Python implementation of import. - -References on import: - - * Language reference - http://docs.python.org/ref/import.html - * __import__ function - http://docs.python.org/lib/built-in-funcs.html - * Packages - http://www.python.org/doc/essays/packages.html - * PEP 235: Import on Case-Insensitive Platforms - http://www.python.org/dev/peps/pep-0235 - * PEP 275: Import Modules from Zip Archives - http://www.python.org/dev/peps/pep-0273 - * PEP 302: New Import Hooks - http://www.python.org/dev/peps/pep-0302/ - * PEP 328: Imports: Multi-line and Absolute/Relative - http://www.python.org/dev/peps/pep-0328 - -""" -__all__ = ['__import__', 'import_module'] - -from . import _bootstrap - -import os -import re -import tokenize +"""A pure Python implementation of import.""" +__all__ = ['__import__', 'import_module', 'invalidate_caches'] # Bootstrap help ##################################################### +import imp +import sys -def _case_ok(directory, check): - """Check if the directory contains something matching 'check'. - - No check is done if the file/directory exists or not. +try: + _bootstrap = sys.modules['_frozen_importlib'] +except ImportError: + from . import _bootstrap + _bootstrap._setup(sys, imp) +else: + # importlib._bootstrap is the built-in import, ensure we don't create + # a second copy of the module. + _bootstrap.__name__ = 'importlib._bootstrap' + _bootstrap.__package__ = 'importlib' + _bootstrap.__file__ = __file__.replace('__init__.py', '_bootstrap.py') + sys.modules['importlib._bootstrap'] = _bootstrap - """ - if 'PYTHONCASEOK' in os.environ: - return True - elif check in os.listdir(directory if directory else os.getcwd()): - return True - return False +# To simplify imports in test code +_w_long = _bootstrap._w_long +_r_long = _bootstrap._r_long -def _w_long(x): - """Convert a 32-bit integer to little-endian. +# Public API ######################################################### - XXX Temporary until marshal's long functions are exposed. +from ._bootstrap import __import__ - """ - x = int(x) - int_bytes = [] - int_bytes.append(x & 0xFF) - int_bytes.append((x >> 8) & 0xFF) - int_bytes.append((x >> 16) & 0xFF) - int_bytes.append((x >> 24) & 0xFF) - return bytearray(int_bytes) +def invalidate_caches(): + """Call the invalidate_caches() method on all finders stored in + sys.path_importer_caches (where implemented).""" + for finder in sys.path_importer_cache.values(): + if hasattr(finder, 'invalidate_caches'): + finder.invalidate_caches() -def _r_long(int_bytes): - """Convert 4 bytes in little-endian to an integer. - XXX Temporary until marshal's long function are exposed. +def find_loader(name, path=None): + """Find the loader for the specified module. - """ - x = int_bytes[0] - x |= int_bytes[1] << 8 - x |= int_bytes[2] << 16 - x |= int_bytes[3] << 24 - return x + First, sys.modules is checked to see if the module was already imported. If + so, then sys.modules[name].__loader__ is returned. If that happens to be + set to None, then ValueError is raised. If the module is not in + sys.modules, then sys.meta_path is searched for a suitable loader with the + value of 'path' given to the finders. None is returned if no loader could + be found. + Dotted names do not have their parent packages implicitly imported. -# Required built-in modules. -try: - import posix as _os -except ImportError: + """ try: - import nt as _os - except ImportError: - try: - import os2 as _os - except ImportError: - raise ImportError('posix, nt, or os2 module required for importlib') -_bootstrap._os = _os -import imp, sys, marshal, errno, _io -_bootstrap.imp = imp -_bootstrap.sys = sys -_bootstrap.marshal = marshal -_bootstrap.errno = errno -_bootstrap._io = _io -import _warnings -_bootstrap._warnings = _warnings - - -from os import sep -# For os.path.join replacement; pull from Include/osdefs.h:SEP . -_bootstrap.path_sep = sep - -_bootstrap._case_ok = _case_ok -marshal._w_long = _w_long -marshal._r_long = _r_long - - -# Public API ######################################################### - -from ._bootstrap import __import__ + loader = sys.modules[name].__loader__ + if loader is None: + raise ValueError('{}.__loader__ is None'.format(name)) + else: + return loader + except KeyError: + pass + return _bootstrap._find_module(name, path) def import_module(name, package=None): diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 90eb1a770f9..36c0e88cace 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -6,33 +6,93 @@ such it requires the injection of specific modules and attributes in order to work. One should use importlib as the public-facing version of this module. """ - -# Injected modules are '_warnings', 'imp', 'sys', 'marshal', 'errno', '_io', -# and '_os' (a.k.a. 'posix', 'nt' or 'os2'). -# Injected attribute is path_sep. # +# IMPORTANT: Whenever making changes to this module, be sure to run +# a top-level make in order to get the frozen version of the module +# update. Not doing so, will result in the Makefile to fail for +# all others who don't have a ./python around to freeze the module +# in the early stages of compilation. +# + +# See importlib._setup() for what is injected into the global namespace. + # When editing this code be aware that code executed at import time CANNOT # reference any injected objects! This includes not only global code but also # anything specified at the class level. +# XXX Make sure all public names have no single leading underscore and all +# others do. + # Bootstrap-related code ###################################################### -# XXX Could also expose Modules/getpath.c:joinpath() -def _path_join(*args): - """Replacement for os.path.join.""" - return path_sep.join(x[:-len(path_sep)] if x.endswith(path_sep) else x - for x in args if x) +_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin' -def _path_exists(path): - """Replacement for os.path.exists.""" - try: - _os.stat(path) - except OSError: - return False +def _make_relax_case(): + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): + def _relax_case(): + """True if filenames must be checked case-insensitively.""" + return b'PYTHONCASEOK' in _os.environ else: - return True + def _relax_case(): + """True if filenames must be checked case-insensitively.""" + return False + return _relax_case + + +# TODO: Expose from marshal +def _w_long(x): + """Convert a 32-bit integer to little-endian. + + XXX Temporary until marshal's long functions are exposed. + + """ + x = int(x) + int_bytes = [] + int_bytes.append(x & 0xFF) + int_bytes.append((x >> 8) & 0xFF) + int_bytes.append((x >> 16) & 0xFF) + int_bytes.append((x >> 24) & 0xFF) + return bytearray(int_bytes) + + +# TODO: Expose from marshal +def _r_long(int_bytes): + """Convert 4 bytes in little-endian to an integer. + + XXX Temporary until marshal's long function are exposed. + + """ + x = int_bytes[0] + x |= int_bytes[1] << 8 + x |= int_bytes[2] << 16 + x |= int_bytes[3] << 24 + return x + + +def _path_join(*path_parts): + """Replacement for os.path.join().""" + new_parts = [] + for part in path_parts: + if not part: + continue + new_parts.append(part) + if part[-1] not in path_separators: + new_parts.append(path_sep) + return ''.join(new_parts[:-1]) # Drop superfluous path separator. + + +def _path_split(path): + """Replacement for os.path.split().""" + for x in reversed(path): + if x in path_separators: + sep = x + break + else: + sep = path_sep + front, _, tail = path.rpartition(sep) + return front, tail def _path_is_mode_type(path, mode): @@ -58,61 +118,251 @@ def _path_isdir(path): return _path_is_mode_type(path, 0o040000) -def _path_without_ext(path, ext_type): - """Replacement for os.path.splitext()[0].""" - for suffix in _suffix_list(ext_type): - if path.endswith(suffix): - return path[:-len(suffix)] - else: - raise ValueError("path is not of the specified type") - - -def _path_absolute(path): - """Replacement for os.path.abspath.""" - if not path: - path = _os.getcwd() +def _write_atomic(path, data): + """Best-effort function to write data to a path atomically. + Be prepared to handle a FileExistsError if concurrent writing of the + temporary file is attempted.""" + # id() is used to generate a pseudo-random filename. + path_tmp = '{}.{}'.format(path, id(path)) + fd = _os.open(path_tmp, _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, 0o666) try: - return _os._getfullpathname(path) - except AttributeError: - if path.startswith('/'): - return path - else: - return _path_join(_os.getcwd(), path) + # We first write data to a temporary file, and then use os.replace() to + # perform an atomic rename. + with _io.FileIO(fd, 'wb') as file: + file.write(data) + _os.replace(path_tmp, path) + except OSError: + try: + _os.unlink(path_tmp) + except OSError: + pass + raise def _wrap(new, old): """Simple substitute for functools.wraps.""" - for replace in ['__module__', '__name__', '__doc__']: - setattr(new, replace, getattr(old, replace)) + for replace in ['__module__', '__name__', '__qualname__', '__doc__']: + if hasattr(old, replace): + setattr(new, replace, getattr(old, replace)) new.__dict__.update(old.__dict__) -code_type = type(_wrap.__code__) +_code_type = type(_wrap.__code__) + + +def new_module(name): + """Create a new module. + + The module is not entered into sys.modules. + + """ + return type(_io)(name) + + +# Module-level locking ######################################################## + +# A dict mapping module names to weakrefs of _ModuleLock instances +_module_locks = {} +# A dict mapping thread ids to _ModuleLock instances +_blocking_on = {} + + +class _DeadlockError(RuntimeError): + pass + + +class _ModuleLock: + """A recursive lock implementation which is able to detect deadlocks + (e.g. thread 1 trying to take locks A then B, and thread 2 trying to + take locks B then A). + """ + + def __init__(self, name): + self.lock = _thread.allocate_lock() + self.wakeup = _thread.allocate_lock() + self.name = name + self.owner = None + self.count = 0 + self.waiters = 0 + + def has_deadlock(self): + # Deadlock avoidance for concurrent circular imports. + me = _thread.get_ident() + tid = self.owner + while True: + lock = _blocking_on.get(tid) + if lock is None: + return False + tid = lock.owner + if tid == me: + return True + + def acquire(self): + """ + Acquire the module lock. If a potential deadlock is detected, + a _DeadlockError is raised. + Otherwise, the lock is always acquired and True is returned. + """ + tid = _thread.get_ident() + _blocking_on[tid] = self + try: + while True: + with self.lock: + if self.count == 0 or self.owner == tid: + self.owner = tid + self.count += 1 + return True + if self.has_deadlock(): + raise _DeadlockError("deadlock detected by %r" % self) + if self.wakeup.acquire(False): + self.waiters += 1 + # Wait for a release() call + self.wakeup.acquire() + self.wakeup.release() + finally: + del _blocking_on[tid] + + def release(self): + tid = _thread.get_ident() + with self.lock: + if self.owner != tid: + raise RuntimeError("cannot release un-acquired lock") + assert self.count > 0 + self.count -= 1 + if self.count == 0: + self.owner = None + if self.waiters: + self.waiters -= 1 + self.wakeup.release() + + def __repr__(self): + return "_ModuleLock(%r) at %d" % (self.name, id(self)) + + +class _DummyModuleLock: + """A simple _ModuleLock equivalent for Python builds without + multi-threading support.""" + + def __init__(self, name): + self.name = name + self.count = 0 + + def acquire(self): + self.count += 1 + return True + + def release(self): + if self.count == 0: + raise RuntimeError("cannot release un-acquired lock") + self.count -= 1 + + def __repr__(self): + return "_DummyModuleLock(%r) at %d" % (self.name, id(self)) + + +# The following two functions are for consumption by Python/import.c. + +def _get_module_lock(name): + """Get or create the module lock for a given module name. + + Should only be called with the import lock taken.""" + lock = None + if name in _module_locks: + lock = _module_locks[name]() + if lock is None: + if _thread is None: + lock = _DummyModuleLock(name) + else: + lock = _ModuleLock(name) + def cb(_): + del _module_locks[name] + _module_locks[name] = _weakref.ref(lock, cb) + return lock + +def _lock_unlock_module(name): + """Release the global import lock, and acquires then release the + module lock for a given module name. + This is used to ensure a module is completely initialized, in the + event it is being imported by another thread. + + Should only be called with the import lock taken.""" + lock = _get_module_lock(name) + _imp.release_lock() + try: + lock.acquire() + except _DeadlockError: + # Concurrent circular import, we'll accept a partially initialized + # module object. + pass + else: + lock.release() + # Finder/loader utility code ################################################## +_PYCACHE = '__pycache__' + +SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. + +DEBUG_BYTECODE_SUFFIXES = ['.pyc'] +OPTIMIZED_BYTECODE_SUFFIXES = ['.pyo'] +if __debug__: + BYTECODE_SUFFIXES = DEBUG_BYTECODE_SUFFIXES +else: + BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES + +def cache_from_source(path, debug_override=None): + """Given the path to a .py file, return the path to its .pyc/.pyo file. + + The .py file does not need to exist; this simply returns the path to the + .pyc/.pyo file calculated as if the .py file were imported. The extension + will be .pyc unless __debug__ is not defined, then it will be .pyo. + + If debug_override is not None, then it must be a boolean and is taken as + the value of __debug__ instead. + + """ + debug = __debug__ if debug_override is None else debug_override + if debug: + suffixes = DEBUG_BYTECODE_SUFFIXES + else: + suffixes = OPTIMIZED_BYTECODE_SUFFIXES + head, tail = _path_split(path) + base_filename, sep, _ = tail.partition('.') + filename = ''.join([base_filename, sep, _TAG, suffixes[0]]) + return _path_join(head, _PYCACHE, filename) + + +def _verbose_message(message, *args): + """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" + if sys.flags.verbose: + if not message.startswith(('#', 'import ')): + message = '# ' + message + print(message.format(*args), file=sys.stderr) + + def set_package(fxn): """Set __package__ on the returned module.""" - def wrapper(*args, **kwargs): + def set_package_wrapper(*args, **kwargs): module = fxn(*args, **kwargs) if not hasattr(module, '__package__') or module.__package__ is None: module.__package__ = module.__name__ if not hasattr(module, '__path__'): module.__package__ = module.__package__.rpartition('.')[0] return module - _wrap(wrapper, fxn) - return wrapper + _wrap(set_package_wrapper, fxn) + return set_package_wrapper def set_loader(fxn): """Set __loader__ on the returned module.""" - def wrapper(self, *args, **kwargs): + def set_loader_wrapper(self, *args, **kwargs): module = fxn(self, *args, **kwargs) if not hasattr(module, '__loader__'): module.__loader__ = self return module - _wrap(wrapper, fxn) - return wrapper + _wrap(set_loader_wrapper, fxn) + return set_loader_wrapper def module_for_loader(fxn): @@ -120,31 +370,50 @@ def module_for_loader(fxn): The decorated function is passed the module to use instead of the module name. The module passed in to the function is either from sys.modules if - it already exists or is a new module which has __name__ set and is inserted - into sys.modules. If an exception is raised and the decorator created the - module it is subsequently removed from sys.modules. + it already exists or is a new module. If the module is new, then __name__ + is set the first argument to the method, __loader__ is set to self, and + __package__ is set accordingly (if self.is_package() is defined) will be set + before it is passed to the decorated function (if self.is_package() does + not work for the module it will be set post-load). + + If an exception is raised and the decorator created the module it is + subsequently removed from sys.modules. The decorator assumes that the decorated function takes the module name as the second argument. """ - def decorated(self, fullname, *args, **kwargs): + def module_for_loader_wrapper(self, fullname, *args, **kwargs): module = sys.modules.get(fullname) - is_reload = bool(module) + is_reload = module is not None if not is_reload: # This must be done before open() is called as the 'io' module # implicitly imports 'locale' and would otherwise trigger an # infinite loop. - module = imp.new_module(fullname) + module = new_module(fullname) sys.modules[fullname] = module + module.__loader__ = self + try: + is_package = self.is_package(fullname) + except (ImportError, AttributeError): + pass + else: + if is_package: + module.__package__ = fullname + else: + module.__package__ = fullname.rpartition('.')[0] try: + module.__initializing__ = True + # If __package__ was not set above, __import__() will do it later. return fxn(self, module, *args, **kwargs) except: if not is_reload: del sys.modules[fullname] raise - _wrap(decorated, fxn) - return decorated + finally: + module.__initializing__ = False + _wrap(module_for_loader_wrapper, fxn) + return module_for_loader_wrapper def _check_name(method): @@ -155,38 +424,36 @@ def _check_name(method): compared against. If the comparison fails then ImportError is raised. """ - def inner(self, name, *args, **kwargs): - if self._name != name: - raise ImportError("loader cannot handle %s" % name) + def _check_name_wrapper(self, name=None, *args, **kwargs): + if name is None: + name = self.name + elif self.name != name: + raise ImportError("loader cannot handle %s" % name, name=name) return method(self, name, *args, **kwargs) - _wrap(inner, method) - return inner + _wrap(_check_name_wrapper, method) + return _check_name_wrapper def _requires_builtin(fxn): """Decorator to verify the named module is built-in.""" - def wrapper(self, fullname): + def _requires_builtin_wrapper(self, fullname): if fullname not in sys.builtin_module_names: - raise ImportError("{0} is not a built-in module".format(fullname)) + raise ImportError("{0} is not a built-in module".format(fullname), + name=fullname) return fxn(self, fullname) - _wrap(wrapper, fxn) - return wrapper + _wrap(_requires_builtin_wrapper, fxn) + return _requires_builtin_wrapper def _requires_frozen(fxn): """Decorator to verify the named module is frozen.""" - def wrapper(self, fullname): - if not imp.is_frozen(fullname): - raise ImportError("{0} is not a frozen module".format(fullname)) + def _requires_frozen_wrapper(self, fullname): + if not _imp.is_frozen(fullname): + raise ImportError("{0} is not a frozen module".format(fullname), + name=fullname) return fxn(self, fullname) - _wrap(wrapper, fxn) - return wrapper - - -def _suffix_list(suffix_type): - """Return a list of file suffixes based on the imp file type.""" - return [suffix[0] for suffix in imp.get_suffixes() - if suffix[2] == suffix_type] + _wrap(_requires_frozen_wrapper, fxn) + return _requires_frozen_wrapper # Loaders ##################################################################### @@ -201,6 +468,10 @@ class BuiltinImporter: """ @classmethod + def module_repr(cls, module): + return "<module '{}' (built-in)>".format(module.__name__) + + @classmethod def find_module(cls, fullname, path=None): """Find the built-in module. @@ -209,7 +480,7 @@ class BuiltinImporter: """ if path is not None: return None - return cls if imp.is_builtin(fullname) else None + return cls if _imp.is_builtin(fullname) else None @classmethod @set_package @@ -219,7 +490,7 @@ class BuiltinImporter: """Load a built-in module.""" is_reload = fullname in sys.modules try: - return imp.init_builtin(fullname) + return _imp.init_builtin(fullname) except: if not is_reload and fullname in sys.modules: del sys.modules[fullname] @@ -240,7 +511,7 @@ class BuiltinImporter: @classmethod @_requires_builtin def is_package(cls, fullname): - """Return None as built-in module are never packages.""" + """Return None as built-in modules are never packages.""" return False @@ -254,9 +525,13 @@ class FrozenImporter: """ @classmethod + def module_repr(cls, m): + return "<module '{}' (frozen)>".format(m.__name__) + + @classmethod def find_module(cls, fullname, path=None): """Find a frozen module.""" - return cls if imp.is_frozen(fullname) else None + return cls if _imp.is_frozen(fullname) else None @classmethod @set_package @@ -266,7 +541,10 @@ class FrozenImporter: """Load a frozen module.""" is_reload = fullname in sys.modules try: - return imp.init_frozen(fullname) + m = _imp.init_frozen(fullname) + # Let our own module_repr() method produce a suitable repr. + del m.__file__ + return m except: if not is_reload and fullname in sys.modules: del sys.modules[fullname] @@ -276,7 +554,7 @@ class FrozenImporter: @_requires_frozen def get_code(cls, fullname): """Return the code object for the frozen module.""" - return imp.get_frozen_object(fullname) + return _imp.get_frozen_object(fullname) @classmethod @_requires_frozen @@ -288,39 +566,66 @@ class FrozenImporter: @_requires_frozen def is_package(cls, fullname): """Return if the frozen module is a package.""" - return imp.is_frozen_package(fullname) + return _imp.is_frozen_package(fullname) class _LoaderBasics: """Base class of common code needed by both SourceLoader and - _SourcelessFileLoader.""" + SourcelessFileLoader.""" def is_package(self, fullname): """Concrete implementation of InspectLoader.is_package by checking if the path returned by get_filename has a filename of '__init__.py'.""" - filename = self.get_filename(fullname).rpartition(path_sep)[2] - return filename.rsplit('.', 1)[0] == '__init__' + filename = _path_split(self.get_filename(fullname))[1] + filename_base = filename.rsplit('.', 1)[0] + tail_name = fullname.rpartition('.')[2] + return filename_base == '__init__' and tail_name != '__init__' - def _bytes_from_bytecode(self, fullname, data, source_mtime): + def _bytes_from_bytecode(self, fullname, data, bytecode_path, source_stats): """Return the marshalled bytes from bytecode, verifying the magic - number and timestamp along the way. + number, timestamp and source size along the way. - If source_mtime is None then skip the timestamp check. + If source_stats is None then skip the timestamp check. """ magic = data[:4] raw_timestamp = data[4:8] - if len(magic) != 4 or magic != imp.get_magic(): - raise ImportError("bad magic number in {}".format(fullname)) + raw_size = data[8:12] + if magic != _MAGIC_NUMBER: + msg = 'bad magic number in {!r}: {!r}'.format(fullname, magic) + raise ImportError(msg, name=fullname, path=bytecode_path) elif len(raw_timestamp) != 4: - raise EOFError("bad timestamp in {}".format(fullname)) - elif source_mtime is not None: - if marshal._r_long(raw_timestamp) != source_mtime: - raise ImportError("bytecode is stale for {}".format(fullname)) + message = 'bad timestamp in {}'.format(fullname) + _verbose_message(message) + raise EOFError(message) + elif len(raw_size) != 4: + message = 'bad size in {}'.format(fullname) + _verbose_message(message) + raise EOFError(message) + if source_stats is not None: + try: + source_mtime = int(source_stats['mtime']) + except KeyError: + pass + else: + if _r_long(raw_timestamp) != source_mtime: + message = 'bytecode is stale for {}'.format(fullname) + _verbose_message(message) + raise ImportError(message, name=fullname, + path=bytecode_path) + try: + source_size = source_stats['size'] & 0xFFFFFFFF + except KeyError: + pass + else: + if _r_long(raw_size) != source_size: + raise ImportError( + "bytecode is stale for {}".format(fullname), + name=fullname, path=bytecode_path) # Can't return the code object as errors from marshal loading need to # propagate even when source is available. - return data[8:] + return data[12:] @module_for_loader def _load_module(self, module, *, sourceless=False): @@ -330,12 +635,12 @@ class _LoaderBasics: code_object = self.get_code(name) module.__file__ = self.get_filename(name) if not sourceless: - module.__cached__ = imp.cache_from_source(module.__file__) + module.__cached__ = cache_from_source(module.__file__) else: module.__cached__ = module.__file__ module.__package__ = name if self.is_package(name): - module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]] + module.__path__ = [_path_split(module.__file__)[0]] else: module.__package__ = module.__package__.rpartition('.')[0] module.__loader__ = self @@ -348,11 +653,20 @@ class SourceLoader(_LoaderBasics): def path_mtime(self, path): """Optional method that returns the modification time (an int) for the specified path, where path is a str. + """ + raise NotImplementedError - Implementing this method allows the loader to read bytecode files. + def path_stats(self, path): + """Optional method returning a metadata dict for the specified path + to by the path (str). + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + Implementing this method allows the loader to read bytecode files. """ - raise NotImplementedError + return {'mtime': self.path_mtime(path)} def set_data(self, path, data): """Optional method which writes data (bytes) to a file path (a str). @@ -370,7 +684,8 @@ class SourceLoader(_LoaderBasics): try: source_bytes = self.get_data(path) except IOError: - raise ImportError("source not available through get_data()") + raise ImportError("source not available through get_data()", + name=fullname) encoding = tokenize.detect_encoding(_io.BytesIO(source_bytes).readline) newline_decoder = _io.IncrementalNewlineDecoder(None, True) return newline_decoder.decode(source_bytes.decode(encoding[0])) @@ -378,19 +693,20 @@ class SourceLoader(_LoaderBasics): def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. - Reading of bytecode requires path_mtime to be implemented. To write + Reading of bytecode requires path_stats to be implemented. To write bytecode, set_data must also be implemented. """ source_path = self.get_filename(fullname) - bytecode_path = imp.cache_from_source(source_path) + bytecode_path = cache_from_source(source_path) source_mtime = None if bytecode_path is not None: try: - source_mtime = self.path_mtime(source_path) + st = self.path_stats(source_path) except NotImplementedError: pass else: + source_mtime = int(st['mtime']) try: data = self.get_data(bytecode_path) except IOError: @@ -398,29 +714,36 @@ class SourceLoader(_LoaderBasics): else: try: bytes_data = self._bytes_from_bytecode(fullname, data, - source_mtime) + bytecode_path, + st) except (ImportError, EOFError): pass else: + _verbose_message('{} matches {}', bytecode_path, + source_path) found = marshal.loads(bytes_data) - if isinstance(found, code_type): + if isinstance(found, _code_type): + _imp._fix_co_filename(found, source_path) + _verbose_message('code object from {}', + bytecode_path) return found else: msg = "Non-code object in {}" - raise ImportError(msg.format(bytecode_path)) + raise ImportError(msg.format(bytecode_path), + name=fullname, path=bytecode_path) source_bytes = self.get_data(source_path) code_object = compile(source_bytes, source_path, 'exec', dont_inherit=True) + _verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and - source_mtime is not None): - # If e.g. Jython ever implements imp.cache_from_source to have - # their own cached file format, this block of code will most likely - # throw an exception. - data = bytearray(imp.get_magic()) - data.extend(marshal._w_long(source_mtime)) + source_mtime is not None): + data = bytearray(_MAGIC_NUMBER) + data.extend(_w_long(source_mtime)) + data.extend(_w_long(len(source_bytes))) data.extend(marshal.dumps(code_object)) try: self.set_data(bytecode_path, data) + _verbose_message('wrote {!r}', bytecode_path) except NotImplementedError: pass return code_object @@ -436,7 +759,7 @@ class SourceLoader(_LoaderBasics): return self._load_module(fullname) -class _FileLoader: +class FileLoader: """Base file loader class which implements the loader protocol methods that require file system usage.""" @@ -444,13 +767,20 @@ class _FileLoader: def __init__(self, fullname, path): """Cache the module name and the path to the file found by the finder.""" - self._name = fullname - self._path = path + self.name = fullname + self.path = path + + @_check_name + def load_module(self, fullname): + """Load a module from a file.""" + # Issue #14857: Avoid the zero-argument form so the implementation + # of that form can be updated without breaking the frozen module + return super(FileLoader, self).load_module(fullname) @_check_name def get_filename(self, fullname): """Return the path to the source file as found by the finder.""" - return self._path + return self.path def get_data(self, path): """Return the data from path as raw bytes.""" @@ -458,52 +788,45 @@ class _FileLoader: return file.read() -class _SourceFileLoader(_FileLoader, SourceLoader): +class SourceFileLoader(FileLoader, SourceLoader): """Concrete implementation of SourceLoader using the file system.""" - def path_mtime(self, path): - """Return the modification time for the path.""" - return int(_os.stat(path).st_mtime) + def path_stats(self, path): + """Return the metadat for the path.""" + st = _os.stat(path) + return {'mtime': st.st_mtime, 'size': st.st_size} def set_data(self, path, data): """Write bytes data to a file.""" - parent, _, filename = path.rpartition(path_sep) + parent, filename = _path_split(path) path_parts = [] # Figure out what directories are missing. while parent and not _path_isdir(parent): - parent, _, part = parent.rpartition(path_sep) + parent, part = _path_split(parent) path_parts.append(part) # Create needed directories. for part in reversed(path_parts): parent = _path_join(parent, part) try: _os.mkdir(parent) - except OSError as exc: + except FileExistsError: # Probably another Python process already created the dir. - if exc.errno == errno.EEXIST: - continue - else: - raise - except IOError as exc: + continue + except PermissionError: # If can't get proper access, then just forget about writing # the data. - if exc.errno == errno.EACCES: - return - else: - raise - try: - with _io.FileIO(path, 'wb') as file: - file.write(data) - except IOError as exc: - # Don't worry if you can't write bytecode. - if exc.errno == errno.EACCES: return - else: - raise + try: + _write_atomic(path, data) + _verbose_message('created {!r}', path) + except (PermissionError, FileExistsError): + # Don't worry if you can't write bytecode or someone is writing + # it at the same time. + pass -class _SourcelessFileLoader(_FileLoader, _LoaderBasics): +class SourcelessFileLoader(FileLoader, _LoaderBasics): """Loader which handles sourceless file imports.""" @@ -513,19 +836,21 @@ class _SourcelessFileLoader(_FileLoader, _LoaderBasics): def get_code(self, fullname): path = self.get_filename(fullname) data = self.get_data(path) - bytes_data = self._bytes_from_bytecode(fullname, data, None) + bytes_data = self._bytes_from_bytecode(fullname, data, path, None) found = marshal.loads(bytes_data) - if isinstance(found, code_type): + if isinstance(found, _code_type): + _verbose_message('code object from {!r}', path) return found else: - raise ImportError("Non-code object in {}".format(path)) + raise ImportError("Non-code object in {}".format(path), + name=fullname, path=path) def get_source(self, fullname): """Return None as there is no source code.""" return None -class _ExtensionFileLoader: +class ExtensionFileLoader: """Loader for extension modules. @@ -534,14 +859,8 @@ class _ExtensionFileLoader: """ def __init__(self, name, path): - """Initialize the loader. - - If is_pkg is True then an exception is raised as extension modules - cannot be the __init__ module for an extension module. - - """ - self._name = name - self._path = path + self.name = name + self.path = path @_check_name @set_package @@ -550,28 +869,98 @@ class _ExtensionFileLoader: """Load an extension module.""" is_reload = fullname in sys.modules try: - return imp.load_dynamic(fullname, self._path) + module = _imp.load_dynamic(fullname, self.path) + _verbose_message('extension module loaded from {!r}', self.path) + return module except: if not is_reload and fullname in sys.modules: del sys.modules[fullname] raise - @_check_name def is_package(self, fullname): """Return False as an extension module can never be a package.""" return False - @_check_name def get_code(self, fullname): """Return None as an extension module cannot create a code object.""" return None - @_check_name def get_source(self, fullname): """Return None as extension modules have no source code.""" return None +class _NamespacePath: + """Represents a namespace package's path. It uses the module name + to find its parent module, and from there it looks up the parent's + __path__. When this changes, the module's own path is recomputed, + using path_finder. For top-leve modules, the parent module's path + is sys.path.""" + + def __init__(self, name, path, path_finder): + self._name = name + self._path = path + self._last_parent_path = tuple(self._get_parent_path()) + self._path_finder = path_finder + + def _find_parent_path_names(self): + """Returns a tuple of (parent-module-name, parent-path-attr-name)""" + parent, dot, me = self._name.rpartition('.') + if dot == '': + # This is a top-level module. sys.path contains the parent path. + return 'sys', 'path' + # Not a top-level module. parent-module.__path__ contains the + # parent path. + return parent, '__path__' + + def _get_parent_path(self): + parent_module_name, path_attr_name = self._find_parent_path_names() + return getattr(sys.modules[parent_module_name], path_attr_name) + + def _recalculate(self): + # If the parent's path has changed, recalculate _path + parent_path = tuple(self._get_parent_path()) # Make a copy + if parent_path != self._last_parent_path: + loader, new_path = self._path_finder(self._name, parent_path) + # Note that no changes are made if a loader is returned, but we + # do remember the new parent path + if loader is None: + self._path = new_path + self._last_parent_path = parent_path # Save the copy + return self._path + + def __iter__(self): + return iter(self._recalculate()) + + def __len__(self): + return len(self._recalculate()) + + def __repr__(self): + return "_NamespacePath({0!r})".format(self._path) + + def __contains__(self, item): + return item in self._recalculate() + + def append(self, item): + self._path.append(item) + + +class NamespaceLoader: + def __init__(self, name, path, path_finder): + self._path = _NamespacePath(name, path, path_finder) + + @classmethod + def module_repr(cls, module): + return "<module '{}' (namespace)>".format(module.__name__) + + @module_for_loader + def load_module(self, module): + """Load a namespace module.""" + _verbose_message('namespace module loaded with path {!r}', self._path) + module.__path__ = self._path + return module + + # Finders ##################################################################### class PathFinder: @@ -579,265 +968,395 @@ class PathFinder: """Meta path finder for sys.(path|path_hooks|path_importer_cache).""" @classmethod - def _path_hooks(cls, path, hooks=None): + def _path_hooks(cls, path): """Search sequence of hooks for a finder for 'path'. If 'hooks' is false then use sys.path_hooks. """ - if not hooks: - hooks = sys.path_hooks - for hook in hooks: + if not sys.path_hooks: + _warnings.warn('sys.path_hooks is empty', ImportWarning) + for hook in sys.path_hooks: try: return hook(path) except ImportError: continue else: - raise ImportError("no path hook found for {0}".format(path)) + return None @classmethod - def _path_importer_cache(cls, path, default=None): + def _path_importer_cache(cls, path): """Get the finder for the path from sys.path_importer_cache. If the path is not in the cache, find the appropriate finder and cache - it. If None is cached, get the default finder and cache that - (if applicable). - - Because of NullImporter, some finder should be returned. The only - explicit fail case is if None is cached but the path cannot be used for - the default hook, for which ImportError is raised. + it. If no finder is available, store None. """ + if path == '': + path = '.' try: finder = sys.path_importer_cache[path] except KeyError: finder = cls._path_hooks(path) sys.path_importer_cache[path] = finder - else: - if finder is None and default: - # Raises ImportError on failure. - finder = default(path) - sys.path_importer_cache[path] = finder return finder @classmethod + def _get_loader(cls, fullname, path): + """Find the loader or namespace_path for this module/package name.""" + # If this ends up being a namespace package, namespace_path is + # the list of paths that will become its __path__ + namespace_path = [] + for entry in path: + finder = cls._path_importer_cache(entry) + if finder is not None: + if hasattr(finder, 'find_loader'): + loader, portions = finder.find_loader(fullname) + else: + loader = finder.find_module(fullname) + portions = [] + if loader is not None: + # We found a loader: return it immediately. + return (loader, namespace_path) + # This is possibly part of a namespace package. + # Remember these path entries (if any) for when we + # create a namespace package, and continue iterating + # on path. + namespace_path.extend(portions) + else: + return (None, namespace_path) + + @classmethod def find_module(cls, fullname, path=None): """Find the module on sys.path or 'path' based on sys.path_hooks and sys.path_importer_cache.""" - if not path: + if path is None: path = sys.path - for entry in path: - try: - finder = cls._path_importer_cache(entry) - except ImportError: - continue - if finder: - loader = finder.find_module(fullname) - if loader: - return loader + loader, namespace_path = cls._get_loader(fullname, path) + if loader is not None: + return loader else: - return None + if namespace_path: + # We found at least one namespace path. Return a + # loader which can create the namespace package. + return NamespaceLoader(fullname, namespace_path, cls._get_loader) + else: + return None -class _FileFinder: +class FileFinder: """File-based finder. - Constructor takes a list of objects detailing what file extensions their - loader supports along with whether it can be used for a package. + Interactions with the file system are cached for performance, being + refreshed when the directory the finder is handling has been modified. """ def __init__(self, path, *details): - """Initialize with finder details.""" + """Initialize with the path to search on and a variable number of + 3-tuples containing the loader, file suffixes the loader recognizes, + and a boolean of whether the loader handles packages.""" packages = [] modules = [] - for detail in details: - modules.extend((suffix, detail.loader) for suffix in detail.suffixes) - if detail.supports_packages: - packages.extend((suffix, detail.loader) - for suffix in detail.suffixes) + for loader, suffixes, supports_packages in details: + modules.extend((suffix, loader) for suffix in suffixes) + if supports_packages: + packages.extend((suffix, loader) for suffix in suffixes) self.packages = packages self.modules = modules - self.path = path + # Base (directory) path + self.path = path or '.' + self._path_mtime = -1 + self._path_cache = set() + self._relaxed_path_cache = set() + + def invalidate_caches(self): + """Invalidate the directory mtime.""" + self._path_mtime = -1 def find_module(self, fullname): """Try to find a loader for the specified module.""" + # Call find_loader(). If it returns a string (indicating this + # is a namespace package portion), generate a warning and + # return None. + loader, portions = self.find_loader(fullname) + assert len(portions) in [0, 1] + if loader is None and len(portions): + msg = "Not importing directory {}: missing __init__" + _warnings.warn(msg.format(portions[0]), ImportWarning) + return loader + + def find_loader(self, fullname): + """Try to find a loader for the specified module, or the namespace + package portions. Returns (loader, list-of-portions).""" + is_namespace = False tail_module = fullname.rpartition('.')[2] - base_path = _path_join(self.path, tail_module) - if _path_isdir(base_path) and _case_ok(self.path, tail_module): - for suffix, loader in self.packages: - init_filename = '__init__' + suffix - full_path = _path_join(base_path, init_filename) - if (_path_isfile(full_path) and - _case_ok(base_path, init_filename)): - return loader(fullname, full_path) - else: - msg = "Not importing directory {}: missing __init__" - _warnings.warn(msg.format(base_path), ImportWarning) + try: + mtime = _os.stat(self.path).st_mtime + except OSError: + mtime = -1 + if mtime != self._path_mtime: + self._fill_cache() + self._path_mtime = mtime + # tail_module keeps the original casing, for __file__ and friends + if _relax_case(): + cache = self._relaxed_path_cache + cache_module = tail_module.lower() + else: + cache = self._path_cache + cache_module = tail_module + # Check if the module is the name of a directory (and thus a package). + if cache_module in cache: + base_path = _path_join(self.path, tail_module) + if _path_isdir(base_path): + for suffix, loader in self.packages: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if _path_isfile(full_path): + return (loader(fullname, full_path), [base_path]) + else: + # A namespace package, return the path if we don't also + # find a module in the next section. + is_namespace = True + # Check for a file w/ a proper suffix exists. for suffix, loader in self.modules: - mod_filename = tail_module + suffix - full_path = _path_join(self.path, mod_filename) - if _path_isfile(full_path) and _case_ok(self.path, mod_filename): - return loader(fullname, full_path) - return None - -class _SourceFinderDetails: - - loader = _SourceFileLoader - supports_packages = True - - def __init__(self): - self.suffixes = _suffix_list(imp.PY_SOURCE) - -class _SourcelessFinderDetails: - - loader = _SourcelessFileLoader - supports_packages = True + if cache_module + suffix in cache: + full_path = _path_join(self.path, tail_module + suffix) + if _path_isfile(full_path): + return (loader(fullname, full_path), []) + if is_namespace: + return (None, [base_path]) + return (None, []) + + def _fill_cache(self): + """Fill the cache of potential modules and packages for this directory.""" + path = self.path + contents = _os.listdir(path) + # We store two cached versions, to handle runtime changes of the + # PYTHONCASEOK environment variable. + if not sys.platform.startswith('win'): + self._path_cache = set(contents) + else: + # Windows users can import modules with case-insensitive file + # suffixes (for legacy reasons). Make the suffix lowercase here + # so it's done once instead of for every import. This is safe as + # the specified suffixes to check against are always specified in a + # case-sensitive manner. + lower_suffix_contents = set() + for item in contents: + name, dot, suffix = item.partition('.') + if dot: + new_name = '{}.{}'.format(name, suffix.lower()) + else: + new_name = name + lower_suffix_contents.add(new_name) + self._path_cache = lower_suffix_contents + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): + self._relaxed_path_cache = set(fn.lower() for fn in contents) - def __init__(self): - self.suffixes = _suffix_list(imp.PY_COMPILED) + @classmethod + def path_hook(cls, *loader_details): + """A class method which returns a closure to use on sys.path_hook + which will return an instance using the specified loaders and the path + called on the closure. + If the path called on the closure is not a directory, ImportError is + raised. -class _ExtensionFinderDetails: + """ + def path_hook_for_FileFinder(path): + """Path hook for importlib.machinery.FileFinder.""" + if not _path_isdir(path): + raise ImportError("only directories are supported", path=path) + return cls(path, *loader_details) - loader = _ExtensionFileLoader - supports_packages = False + return path_hook_for_FileFinder - def __init__(self): - self.suffixes = _suffix_list(imp.C_EXTENSION) + def __repr__(self): + return "FileFinder(%r)" % (self.path,) # Import itself ############################################################### -def _file_path_hook(path): - """If the path is a directory, return a file-based finder.""" - if _path_isdir(path): - return _FileFinder(path, _ExtensionFinderDetails(), - _SourceFinderDetails(), - _SourcelessFinderDetails()) - else: - raise ImportError("only directories are supported") - - -_DEFAULT_PATH_HOOK = _file_path_hook - -class _DefaultPathFinder(PathFinder): - - """Subclass of PathFinder that implements implicit semantics for - __import__.""" - - @classmethod - def _path_hooks(cls, path): - """Search sys.path_hooks as well as implicit path hooks.""" - try: - return super()._path_hooks(path) - except ImportError: - implicit_hooks = [_DEFAULT_PATH_HOOK, imp.NullImporter] - return super()._path_hooks(path, implicit_hooks) - - @classmethod - def _path_importer_cache(cls, path): - """Use the default path hook when None is stored in - sys.path_importer_cache.""" - return super()._path_importer_cache(path, _DEFAULT_PATH_HOOK) - - class _ImportLockContext: """Context manager for the import lock.""" def __enter__(self): """Acquire the import lock.""" - imp.acquire_lock() + _imp.acquire_lock() def __exit__(self, exc_type, exc_value, exc_traceback): """Release the import lock regardless of any raised exceptions.""" - imp.release_lock() + _imp.release_lock() -_IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder] +def _resolve_name(name, package, level): + """Resolve a relative module name to an absolute one.""" + bits = package.rsplit('.', level - 1) + if len(bits) < level: + raise ValueError('attempted relative import beyond top-level package') + base = bits[0] + return '{0}.{1}'.format(base, name) if name else base -_ERR_MSG = 'No module named {}' -def _gcd_import(name, package=None, level=0): - """Import and return the module based on its name, the package the call is - being made from, and the level adjustment. +def _find_module(name, path): + """Find a module's loader.""" + if not sys.meta_path: + _warnings.warn('sys.meta_path is empty', ImportWarning) + for finder in sys.meta_path: + with _ImportLockContext(): + loader = finder.find_module(name, path) + if loader is not None: + # The parent import may have already imported this module. + if name not in sys.modules: + return loader + else: + return sys.modules[name].__loader__ + else: + return None - This function represents the greatest common denominator of functionality - between import_module and __import__. This includes settting __package__ if - the loader did not. - """ +def _sanity_check(name, package, level): + """Verify arguments are "sane".""" + if not isinstance(name, str): + raise TypeError("module name must be str, not {}".format(type(name))) + if level < 0: + raise ValueError('level must be >= 0') if package: - if not hasattr(package, 'rindex'): - raise ValueError("__package__ not set to a string") + if not isinstance(package, str): + raise TypeError("__package__ not set to a string") elif package not in sys.modules: msg = ("Parent module {0!r} not loaded, cannot perform relative " "import") raise SystemError(msg.format(package)) if not name and level == 0: raise ValueError("Empty module name") - if level > 0: - dot = len(package) - for x in range(level, 1, -1): - try: - dot = package.rindex('.', 0, dot) - except ValueError: - raise ValueError("attempted relative import beyond " - "top-level package") - if name: - name = "{0}.{1}".format(package[:dot], name) - else: - name = package[:dot] - with _ImportLockContext(): + + +_ERR_MSG = 'No module named {!r}' + +def _find_and_load_unlocked(name, import_): + path = None + parent = name.rpartition('.')[0] + if parent: + if parent not in sys.modules: + import_(parent) + # Crazy side-effects! + if name in sys.modules: + return sys.modules[name] + # Backwards-compatibility; be nicer to skip the dict lookup. + parent_module = sys.modules[parent] try: - module = sys.modules[name] - if module is None: - message = ("import of {} halted; " - "None in sys.modules".format(name)) - raise ImportError(message) - return module - except KeyError: + path = parent_module.__path__ + except AttributeError: + msg = (_ERR_MSG + '; {} is not a package').format(name, parent) + raise ImportError(msg, name=name) + loader = _find_module(name, path) + if loader is None: + raise ImportError(_ERR_MSG.format(name), name=name) + elif name not in sys.modules: + # The parent import may have already imported this module. + loader.load_module(name) + _verbose_message('import {!r} # {!r}', name, loader) + # Backwards-compatibility; be nicer to skip the dict lookup. + module = sys.modules[name] + if parent: + # Set the module as an attribute on its parent. + parent_module = sys.modules[parent] + setattr(parent_module, name.rpartition('.')[2], module) + # Set __package__ if the loader did not. + if not hasattr(module, '__package__') or module.__package__ is None: + try: + module.__package__ = module.__name__ + if not hasattr(module, '__path__'): + module.__package__ = module.__package__.rpartition('.')[0] + except AttributeError: pass - parent = name.rpartition('.')[0] - path = None - if parent: - if parent not in sys.modules: - _gcd_import(parent) - # Backwards-compatibility; be nicer to skip the dict lookup. - parent_module = sys.modules[parent] - try: - path = parent_module.__path__ - except AttributeError: - msg = (_ERR_MSG + '; {} is not a package').format(name, parent) - raise ImportError(msg) - meta_path = sys.meta_path + _IMPLICIT_META_PATH - for finder in meta_path: - loader = finder.find_module(name, path) - if loader is not None: - # The parent import may have already imported this module. - if name not in sys.modules: - loader.load_module(name) - break - else: - raise ImportError(_ERR_MSG.format(name)) - # Backwards-compatibility; be nicer to skip the dict lookup. - module = sys.modules[name] - if parent: - # Set the module as an attribute on its parent. - setattr(parent_module, name.rpartition('.')[2], module) - # Set __package__ if the loader did not. - if not hasattr(module, '__package__') or module.__package__ is None: - # Watch out for what comes out of sys.modules to not be a module, - # e.g. an int. + # Set loader if need be. + if not hasattr(module, '__loader__'): + try: + module.__loader__ = loader + except AttributeError: + pass + return module + + +def _find_and_load(name, import_): + """Find and load the module, and release the import lock.""" + try: + lock = _get_module_lock(name) + finally: + _imp.release_lock() + lock.acquire() + try: + return _find_and_load_unlocked(name, import_) + finally: + lock.release() + + +def _gcd_import(name, package=None, level=0): + """Import and return the module based on its name, the package the call is + being made from, and the level adjustment. + + This function represents the greatest common denominator of functionality + between import_module and __import__. This includes setting __package__ if + the loader did not. + + """ + _sanity_check(name, package, level) + if level > 0: + name = _resolve_name(name, package, level) + _imp.acquire_lock() + if name not in sys.modules: + return _find_and_load(name, _gcd_import) + module = sys.modules[name] + if module is None: + _imp.release_lock() + message = ("import of {} halted; " + "None in sys.modules".format(name)) + raise ImportError(message, name=name) + _lock_unlock_module(name) + return module + + +def _handle_fromlist(module, fromlist, import_): + """Figure out what __import__ should return. + + The import_ parameter is a callable which takes the name of module to + import. It is required to decouple the function from assuming importlib's + import implementation is desired. + + """ + # The hell that is fromlist ... + # If a package was imported, try to import stuff from fromlist. + if hasattr(module, '__path__'): + if '*' in fromlist and hasattr(module, '__all__'): + fromlist = list(fromlist) + fromlist.remove('*') + fromlist.extend(module.__all__) + for x in (y for y in fromlist if not hasattr(module, y)): try: - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = module.__package__.rpartition('.')[0] - except AttributeError: + import_('{0}.{1}'.format(module.__name__, x)) + except ImportError: pass - return module + return module + + +def _calc___package__(globals): + """Calculate what __package__ should be. + + __package__ is not guaranteed to be defined or could be set to None + to represent that its proper value is unknown. + + """ + package = globals.get('__package__') + if package is None: + package = globals['__name__'] + if '__path__' not in globals: + package = package.rpartition('.')[0] + return package def __import__(name, globals={}, locals={}, fromlist=[], level=0): @@ -851,40 +1370,99 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0): import (e.g. ``from ..pkg import mod`` would have a 'level' of 2). """ - if not hasattr(name, 'rpartition'): - raise TypeError("module name must be str, not {}".format(type(name))) if level == 0: module = _gcd_import(name) else: - # __package__ is not guaranteed to be defined or could be set to None - # to represent that it's proper value is unknown - package = globals.get('__package__') - if package is None: - package = globals['__name__'] - if '__path__' not in globals: - package = package.rpartition('.')[0] + package = _calc___package__(globals) module = _gcd_import(name, package, level) - # The hell that is fromlist ... if not fromlist: # Return up to the first dot in 'name'. This is complicated by the fact # that 'name' may be relative. if level == 0: - return sys.modules[name.partition('.')[0]] + return _gcd_import(name.partition('.')[0]) elif not name: return module else: cut_off = len(name) - len(name.partition('.')[0]) - return sys.modules[module.__name__[:-cut_off]] + return sys.modules[module.__name__[:len(module.__name__)-cut_off]] else: - # If a package was imported, try to import stuff from fromlist. - if hasattr(module, '__path__'): - if '*' in fromlist and hasattr(module, '__all__'): - fromlist = list(fromlist) - fromlist.remove('*') - fromlist.extend(module.__all__) - for x in (y for y in fromlist if not hasattr(module,y)): - try: - _gcd_import('{0}.{1}'.format(module.__name__, x)) - except ImportError: - pass - return module + return _handle_fromlist(module, fromlist, _gcd_import) + + +_MAGIC_NUMBER = None # Set in _setup() +_TAG = None # Set in _setup() + + +def _setup(sys_module, _imp_module): + """Setup importlib by importing needed built-in modules and injecting them + into the global namespace. + + As sys is needed for sys.modules access and _imp is needed to load built-in + modules, those two modules must be explicitly passed in. + + """ + global _imp, sys + _imp = _imp_module + sys = sys_module + + for module in (_imp, sys): + if not hasattr(module, '__loader__'): + module.__loader__ = BuiltinImporter + + self_module = sys.modules[__name__] + for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'): + if builtin_name not in sys.modules: + builtin_module = BuiltinImporter.load_module(builtin_name) + else: + builtin_module = sys.modules[builtin_name] + setattr(self_module, builtin_name, builtin_module) + + os_details = ('posix', ['/']), ('nt', ['\\', '/']), ('os2', ['\\', '/']) + for builtin_os, path_separators in os_details: + # Assumption made in _path_join() + assert all(len(sep) == 1 for sep in path_separators) + path_sep = path_separators[0] + if builtin_os in sys.modules: + os_module = sys.modules[builtin_os] + break + else: + try: + os_module = BuiltinImporter.load_module(builtin_os) + # TODO: rip out os2 code after 3.3 is released as per PEP 11 + if builtin_os == 'os2' and 'EMX GCC' in sys.version: + path_sep = path_separators[1] + break + except ImportError: + continue + else: + raise ImportError('importlib requires posix or nt') + + try: + thread_module = BuiltinImporter.load_module('_thread') + except ImportError: + # Python was built without threads + thread_module = None + weakref_module = BuiltinImporter.load_module('_weakref') + + setattr(self_module, '_os', os_module) + setattr(self_module, '_thread', thread_module) + setattr(self_module, '_weakref', weakref_module) + setattr(self_module, 'path_sep', path_sep) + setattr(self_module, 'path_separators', set(path_separators)) + # Constants + setattr(self_module, '_relax_case', _make_relax_case()) + setattr(self_module, '_MAGIC_NUMBER', _imp_module.get_magic()) + setattr(self_module, '_TAG', _imp.get_tag()) + if builtin_os == 'nt': + SOURCE_SUFFIXES.append('.pyw') + + +def _install(sys_module, _imp_module): + """Install importlib as the implementation of import.""" + _setup(sys_module, _imp_module) + extensions = ExtensionFileLoader, _imp_module.extension_suffixes(), False + source = SourceFileLoader, SOURCE_SUFFIXES, True + bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True + supported_loaders = [extensions, source, bytecode] + sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) + sys.meta_path.extend([BuiltinImporter, FrozenImporter, PathFinder]) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index fa343f85a47..c171da37aee 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -1,18 +1,28 @@ """Abstract base classes related to import.""" from . import _bootstrap from . import machinery -from . import util +try: + import _frozen_importlib +except ImportError as exc: + if exc.name != '_frozen_importlib': + raise + _frozen_importlib = None import abc import imp -import io import marshal -import os.path import sys import tokenize -import types import warnings +def _register(abstract_cls, *classes): + for cls in classes: + abstract_cls.register(cls) + if _frozen_importlib is not None: + frozen_cls = getattr(_frozen_importlib, cls.__name__) + abstract_cls.register(frozen_cls) + + class Loader(metaclass=abc.ABCMeta): """Abstract base class for import loaders.""" @@ -36,9 +46,8 @@ class Finder(metaclass=abc.ABCMeta): """ raise NotImplementedError -Finder.register(machinery.BuiltinImporter) -Finder.register(machinery.FrozenImporter) -Finder.register(machinery.PathFinder) +_register(Finder, machinery.BuiltinImporter, machinery.FrozenImporter, + machinery.PathFinder, machinery.FileFinder) class ResourceLoader(Loader): @@ -84,8 +93,8 @@ class InspectLoader(Loader): module. The fullname is a str. Returns a str.""" raise NotImplementedError -InspectLoader.register(machinery.BuiltinImporter) -InspectLoader.register(machinery.FrozenImporter) +_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, + machinery.ExtensionFileLoader) class ExecutionLoader(InspectLoader): @@ -104,6 +113,15 @@ class ExecutionLoader(InspectLoader): raise NotImplementedError +class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader): + + """Abstract base class partially implementing the ResourceLoader and + ExecutionLoader ABCs.""" + +_register(FileLoader, machinery.SourceFileLoader, + machinery.SourcelessFileLoader) + + class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader): """Abstract base class for loading source code (and optionally any @@ -123,7 +141,20 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader): def path_mtime(self, path): """Return the (int) modification time for the path (str).""" - raise NotImplementedError + if self.path_stats.__func__ is SourceLoader.path_stats: + raise NotImplementedError + return int(self.path_stats(path)['mtime']) + + def path_stats(self, path): + """Return a metadata dict for the source pointed to by the path (str). + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + """ + if self.path_mtime.__func__ is SourceLoader.path_mtime: + raise NotImplementedError + return {'mtime': self.path_mtime(path)} def set_data(self, path, data): """Write the bytes to the path (if possible). @@ -137,6 +168,7 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader): """ raise NotImplementedError +_register(SourceLoader, machinery.SourceFileLoader) class PyLoader(SourceLoader): @@ -195,10 +227,10 @@ class PyLoader(SourceLoader): "use SourceLoader instead. " "See the importlib documentation on how to be " "compatible with Python 3.1 onwards.", - PendingDeprecationWarning) + DeprecationWarning) path = self.source_path(fullname) if path is None: - raise ImportError + raise ImportError(name=fullname) else: return path @@ -226,7 +258,7 @@ class PyPycLoader(PyLoader): if path is not None: return path raise ImportError("no source or bytecode path available for " - "{0!r}".format(fullname)) + "{0!r}".format(fullname), name=fullname) def get_code(self, fullname): """Get a code object from source or bytecode.""" @@ -234,7 +266,7 @@ class PyPycLoader(PyLoader): "removal in Python 3.4; use SourceLoader instead. " "If Python 3.1 compatibility is required, see the " "latest documentation for PyLoader.", - PendingDeprecationWarning) + DeprecationWarning) source_timestamp = self.source_mtime(fullname) # Try to use bytecode if it is available. bytecode_path = self.bytecode_path(fullname) @@ -243,20 +275,25 @@ class PyPycLoader(PyLoader): try: magic = data[:4] if len(magic) < 4: - raise ImportError("bad magic number in {}".format(fullname)) + raise ImportError( + "bad magic number in {}".format(fullname), + name=fullname, path=bytecode_path) raw_timestamp = data[4:8] if len(raw_timestamp) < 4: raise EOFError("bad timestamp in {}".format(fullname)) - pyc_timestamp = marshal._r_long(raw_timestamp) + pyc_timestamp = _bootstrap._r_long(raw_timestamp) bytecode = data[8:] # Verify that the magic number is valid. if imp.get_magic() != magic: - raise ImportError("bad magic number in {}".format(fullname)) + raise ImportError( + "bad magic number in {}".format(fullname), + name=fullname, path=bytecode_path) # Verify that the bytecode is not stale (only matters when # there is source to fall back on. if source_timestamp: if pyc_timestamp < source_timestamp: - raise ImportError("bytecode is stale") + raise ImportError("bytecode is stale", name=fullname, + path=bytecode_path) except (ImportError, EOFError): # If source is available give it a shot. if source_timestamp is not None: @@ -268,18 +305,19 @@ class PyPycLoader(PyLoader): return marshal.loads(bytecode) elif source_timestamp is None: raise ImportError("no source or bytecode available to create code " - "object for {0!r}".format(fullname)) + "object for {0!r}".format(fullname), + name=fullname) # Use the source. source_path = self.source_path(fullname) if source_path is None: message = "a source path must exist to load {0}".format(fullname) - raise ImportError(message) + raise ImportError(message, name=fullname) source = self.get_data(source_path) code_object = compile(source, source_path, 'exec', dont_inherit=True) # Generate bytecode and write it out. if not sys.dont_write_bytecode: data = bytearray(imp.get_magic()) - data.extend(marshal._w_long(source_timestamp)) + data.extend(_bootstrap._w_long(source_timestamp)) data.extend(marshal.dumps(code_object)) self.write_bytecode(fullname, data) return code_object diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 519774440f1..d5e7250479c 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -1,5 +1,15 @@ """The machinery of importlib: finders, loaders, hooks, etc.""" +import _imp + +from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES) from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter from ._bootstrap import PathFinder +from ._bootstrap import FileFinder +from ._bootstrap import SourceFileLoader +from ._bootstrap import SourcelessFileLoader +from ._bootstrap import ExtensionFileLoader + +EXTENSION_SUFFIXES = _imp.extension_suffixes() diff --git a/Lib/importlib/test/__init__.py b/Lib/importlib/test/__init__.py index e69de29bb2d..815a706c544 100644 --- a/Lib/importlib/test/__init__.py +++ b/Lib/importlib/test/__init__.py @@ -0,0 +1,25 @@ +import os +import sys +import unittest + +def test_suite(package=__package__, directory=os.path.dirname(__file__)): + suite = unittest.TestSuite() + for name in os.listdir(directory): + if name.startswith(('.', '__')): + continue + path = os.path.join(directory, name) + if (os.path.isfile(path) and name.startswith('test_') and + name.endswith('.py')): + submodule_name = os.path.splitext(name)[0] + module_name = "{0}.{1}".format(package, submodule_name) + __import__(module_name, level=0) + module_tests = unittest.findTestCases(sys.modules[module_name]) + suite.addTest(module_tests) + elif os.path.isdir(path): + package_name = "{0}.{1}".format(package, name) + __import__(package_name, level=0) + package_tests = getattr(sys.modules[package_name], 'test_suite')() + suite.addTest(package_tests) + else: + continue + return suite diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py index decc53d8c5a..92171b25caa 100644 --- a/Lib/importlib/test/__main__.py +++ b/Lib/importlib/test/__main__.py @@ -4,26 +4,27 @@ Specifying the ``--builtin`` flag will run tests, where applicable, with builtins.__import__ instead of importlib.__import__. """ -import importlib from importlib.test.import_ import util import os.path from test.support import run_unittest -import sys import unittest def test_main(): - if '__pycache__' in __file__: - parts = __file__.split(os.path.sep) - start_dir = sep.join(parts[:-2]) - else: - start_dir = os.path.dirname(__file__) + start_dir = os.path.dirname(__file__) top_dir = os.path.dirname(os.path.dirname(start_dir)) test_loader = unittest.TestLoader() - if '--builtin' in sys.argv: - util.using___import__ = True run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir)) if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(description='Execute the importlib test ' + 'suite') + parser.add_argument('-b', '--builtin', action='store_true', default=False, + help='use builtins.__import__() instead of importlib') + args = parser.parse_args() + if args.builtin: + util.using___import__ = True test_main() diff --git a/Lib/importlib/test/benchmark.py b/Lib/importlib/test/benchmark.py index b5de6c6b010..183e8180c94 100644 --- a/Lib/importlib/test/benchmark.py +++ b/Lib/importlib/test/benchmark.py @@ -9,9 +9,12 @@ from .source import util as source_util import decimal import imp import importlib +import importlib.machinery +import json import os import py_compile import sys +import tabnanny import timeit @@ -59,13 +62,17 @@ def builtin_mod(seconds, repeat): def source_wo_bytecode(seconds, repeat): - """Source w/o bytecode: simple""" + """Source w/o bytecode: small""" sys.dont_write_bytecode = True try: name = '__importlib_test_benchmark__' # Clears out sys.modules and puts an entry at the front of sys.path. with source_util.create_modules(name) as mapping: assert not os.path.exists(imp.cache_from_source(mapping[name])) + sys.meta_path.append(importlib.machinery.PathFinder) + loader = (importlib.machinery.SourceFileLoader, + importlib.machinery.SOURCE_SUFFIXES, True) + sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds): yield result @@ -73,26 +80,37 @@ def source_wo_bytecode(seconds, repeat): sys.dont_write_bytecode = False -def decimal_wo_bytecode(seconds, repeat): - """Source w/o bytecode: decimal""" - name = 'decimal' - decimal_bytecode = imp.cache_from_source(decimal.__file__) - if os.path.exists(decimal_bytecode): - os.unlink(decimal_bytecode) - sys.dont_write_bytecode = True - try: - for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, - seconds=seconds): - yield result - finally: - sys.dont_write_bytecode = False +def _wo_bytecode(module): + name = module.__name__ + def benchmark_wo_bytecode(seconds, repeat): + """Source w/o bytecode: {}""" + bytecode_path = imp.cache_from_source(module.__file__) + if os.path.exists(bytecode_path): + os.unlink(bytecode_path) + sys.dont_write_bytecode = True + try: + for result in bench(name, lambda: sys.modules.pop(name), + repeat=repeat, seconds=seconds): + yield result + finally: + sys.dont_write_bytecode = False + + benchmark_wo_bytecode.__doc__ = benchmark_wo_bytecode.__doc__.format(name) + return benchmark_wo_bytecode + +tabnanny_wo_bytecode = _wo_bytecode(tabnanny) +decimal_wo_bytecode = _wo_bytecode(decimal) def source_writing_bytecode(seconds, repeat): - """Source writing bytecode: simple""" + """Source writing bytecode: small""" assert not sys.dont_write_bytecode name = '__importlib_test_benchmark__' with source_util.create_modules(name) as mapping: + sys.meta_path.append(importlib.machinery.PathFinder) + loader = (importlib.machinery.SourceFileLoader, + importlib.machinery.SOURCE_SUFFIXES, True) + sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) def cleanup(): sys.modules.pop(name) os.unlink(imp.cache_from_source(mapping[name])) @@ -101,21 +119,33 @@ def source_writing_bytecode(seconds, repeat): yield result -def decimal_writing_bytecode(seconds, repeat): - """Source writing bytecode: decimal""" - assert not sys.dont_write_bytecode - name = 'decimal' - def cleanup(): - sys.modules.pop(name) - os.unlink(imp.cache_from_source(decimal.__file__)) - for result in bench(name, cleanup, repeat=repeat, seconds=seconds): - yield result +def _writing_bytecode(module): + name = module.__name__ + def writing_bytecode_benchmark(seconds, repeat): + """Source writing bytecode: {}""" + assert not sys.dont_write_bytecode + def cleanup(): + sys.modules.pop(name) + os.unlink(imp.cache_from_source(module.__file__)) + for result in bench(name, cleanup, repeat=repeat, seconds=seconds): + yield result + + writing_bytecode_benchmark.__doc__ = ( + writing_bytecode_benchmark.__doc__.format(name)) + return writing_bytecode_benchmark + +tabnanny_writing_bytecode = _writing_bytecode(tabnanny) +decimal_writing_bytecode = _writing_bytecode(decimal) def source_using_bytecode(seconds, repeat): - """Bytecode w/ source: simple""" + """Source w/ bytecode: small""" name = '__importlib_test_benchmark__' with source_util.create_modules(name) as mapping: + sys.meta_path.append(importlib.machinery.PathFinder) + loader = (importlib.machinery.SourceFileLoader, + importlib.machinery.SOURCE_SUFFIXES, True) + sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) py_compile.compile(mapping[name]) assert os.path.exists(imp.cache_from_source(mapping[name])) for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, @@ -123,27 +153,56 @@ def source_using_bytecode(seconds, repeat): yield result -def decimal_using_bytecode(seconds, repeat): - """Bytecode w/ source: decimal""" - name = 'decimal' - py_compile.compile(decimal.__file__) - for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, - seconds=seconds): - yield result +def _using_bytecode(module): + name = module.__name__ + def using_bytecode_benchmark(seconds, repeat): + """Source w/ bytecode: {}""" + py_compile.compile(module.__file__) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + using_bytecode_benchmark.__doc__ = ( + using_bytecode_benchmark.__doc__.format(name)) + return using_bytecode_benchmark -def main(import_): +tabnanny_using_bytecode = _using_bytecode(tabnanny) +decimal_using_bytecode = _using_bytecode(decimal) + + +def main(import_, options): + if options.source_file: + with options.source_file: + prev_results = json.load(options.source_file) + else: + prev_results = {} __builtins__.__import__ = import_ benchmarks = (from_cache, builtin_mod, - source_using_bytecode, source_wo_bytecode, source_writing_bytecode, - decimal_using_bytecode, decimal_writing_bytecode, - decimal_wo_bytecode,) + source_wo_bytecode, source_using_bytecode, + tabnanny_writing_bytecode, + tabnanny_wo_bytecode, tabnanny_using_bytecode, + decimal_writing_bytecode, + decimal_wo_bytecode, decimal_using_bytecode, + ) + if options.benchmark: + for b in benchmarks: + if b.__doc__ == options.benchmark: + benchmarks = [b] + break + else: + print('Unknown benchmark: {!r}'.format(options.benchmark, + file=sys.stderr)) + sys.exit(1) seconds = 1 seconds_plural = 's' if seconds > 1 else '' repeat = 3 - header = "Measuring imports/second over {} second{}, best out of {}\n" - print(header.format(seconds, seconds_plural, repeat)) + header = ('Measuring imports/second over {} second{}, best out of {}\n' + 'Entire benchmark run should take about {} seconds\n' + 'Using {!r} as __import__\n') + print(header.format(seconds, seconds_plural, repeat, + len(benchmarks) * seconds * repeat, __import__)) + new_results = {} for benchmark in benchmarks: print(benchmark.__doc__, "[", end=' ') sys.stdout.flush() @@ -154,19 +213,40 @@ def main(import_): sys.stdout.flush() assert not sys.dont_write_bytecode print("]", "best is", format(max(results), ',d')) + new_results[benchmark.__doc__] = results + if prev_results: + print('\n\nComparing new vs. old\n') + for benchmark in benchmarks: + benchmark_name = benchmark.__doc__ + old_result = max(prev_results[benchmark_name]) + new_result = max(new_results[benchmark_name]) + result = '{:,d} vs. {:,d} ({:%})'.format(new_result, + old_result, + new_result/old_result) + print(benchmark_name, ':', result) + if options.dest_file: + with options.dest_file: + json.dump(new_results, options.dest_file, indent=2) if __name__ == '__main__': - import optparse + import argparse - parser = optparse.OptionParser() - parser.add_option('-b', '--builtin', dest='builtin', action='store_true', + parser = argparse.ArgumentParser() + parser.add_argument('-b', '--builtin', dest='builtin', action='store_true', default=False, help="use the built-in __import__") - options, args = parser.parse_args() - if args: - raise RuntimeError("unrecognized args: {}".format(args)) + parser.add_argument('-r', '--read', dest='source_file', + type=argparse.FileType('r'), + help='file to read benchmark data from to compare ' + 'against') + parser.add_argument('-w', '--write', dest='dest_file', + type=argparse.FileType('w'), + help='file to write benchmark data to') + parser.add_argument('--benchmark', dest='benchmark', + help='specific benchmark to run') + options = parser.parse_args() import_ = __import__ if not options.builtin: import_ = importlib.__import__ - main(import_) + main(import_, options) diff --git a/Lib/importlib/test/builtin/test_loader.py b/Lib/importlib/test/builtin/test_loader.py index 1a8539b1e80..ec126c9ccc9 100644 --- a/Lib/importlib/test/builtin/test_loader.py +++ b/Lib/importlib/test/builtin/test_loader.py @@ -54,15 +54,17 @@ class LoaderTests(abc.LoaderTests): def test_unloadable(self): name = 'dssdsdfff' assert name not in sys.builtin_module_names - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: self.load_module(name) + self.assertEqual(cm.exception.name, name) def test_already_imported(self): # Using the name of a module already imported but not a built-in should # still fail. assert hasattr(importlib, '__file__') # Not a built-in. - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: self.load_module('importlib') + self.assertEqual(cm.exception.name, 'importlib') class InspectLoaderTests(unittest.TestCase): @@ -88,8 +90,9 @@ class InspectLoaderTests(unittest.TestCase): # Modules not built-in should raise ImportError. for meth_name in ('get_code', 'get_source', 'is_package'): method = getattr(machinery.BuiltinImporter, meth_name) - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: method(builtin_util.BAD_NAME) + self.assertRaises(builtin_util.BAD_NAME) diff --git a/Lib/importlib/test/extension/test_case_sensitivity.py b/Lib/importlib/test/extension/test_case_sensitivity.py index e062fb6597d..bdc21e7f99e 100644 --- a/Lib/importlib/test/extension/test_case_sensitivity.py +++ b/Lib/importlib/test/extension/test_case_sensitivity.py @@ -1,3 +1,4 @@ +import imp import sys from test import support import unittest @@ -13,19 +14,27 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase): good_name = ext_util.NAME bad_name = good_name.upper() assert good_name != bad_name - finder = _bootstrap._FileFinder(ext_util.PATH, - _bootstrap._ExtensionFinderDetails()) + finder = _bootstrap.FileFinder(ext_util.PATH, + (_bootstrap.ExtensionFileLoader, + imp.extension_suffixes(), + False)) return finder.find_module(bad_name) def test_case_sensitive(self): with support.EnvironmentVarGuard() as env: env.unset('PYTHONCASEOK') + if b'PYTHONCASEOK' in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') loader = self.find_module() self.assertIsNone(loader) def test_case_insensitivity(self): with support.EnvironmentVarGuard() as env: env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') loader = self.find_module() self.assertTrue(hasattr(loader, 'load_module')) diff --git a/Lib/importlib/test/extension/test_finder.py b/Lib/importlib/test/extension/test_finder.py index ea97483317c..804d8628e21 100644 --- a/Lib/importlib/test/extension/test_finder.py +++ b/Lib/importlib/test/extension/test_finder.py @@ -2,6 +2,7 @@ from importlib import _bootstrap from .. import abc from . import util +import imp import unittest class FinderTests(abc.FinderTests): @@ -9,8 +10,10 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" def find_module(self, fullname): - importer = _bootstrap._FileFinder(util.PATH, - _bootstrap._ExtensionFinderDetails()) + importer = _bootstrap.FileFinder(util.PATH, + (_bootstrap.ExtensionFileLoader, + imp.extension_suffixes(), + False)) return importer.find_module(fullname) def test_module(self): diff --git a/Lib/importlib/test/extension/test_loader.py b/Lib/importlib/test/extension/test_loader.py index 4a783db8a5a..4f486ce6af2 100644 --- a/Lib/importlib/test/extension/test_loader.py +++ b/Lib/importlib/test/extension/test_loader.py @@ -1,4 +1,4 @@ -from importlib import _bootstrap +from importlib import machinery from . import util as ext_util from .. import abc from .. import util @@ -11,10 +11,20 @@ class LoaderTests(abc.LoaderTests): """Test load_module() for extension modules.""" + def setUp(self): + self.loader = machinery.ExtensionFileLoader(ext_util.NAME, + ext_util.FILEPATH) + def load_module(self, fullname): - loader = _bootstrap._ExtensionFileLoader(ext_util.NAME, - ext_util.FILEPATH) - return loader.load_module(fullname) + return self.loader.load_module(fullname) + + def test_load_module_API(self): + # Test the default argument for load_module(). + self.loader.load_module() + self.loader.load_module(None) + with self.assertRaises(ImportError): + self.load_module('XXX') + def test_module(self): with util.uncache(ext_util.NAME): @@ -25,7 +35,7 @@ class LoaderTests(abc.LoaderTests): self.assertEqual(getattr(module, attr), value) self.assertTrue(ext_util.NAME in sys.modules) self.assertTrue(isinstance(module.__loader__, - _bootstrap._ExtensionFileLoader)) + machinery.ExtensionFileLoader)) def test_package(self): # Extensions are not found in packages. @@ -46,8 +56,10 @@ class LoaderTests(abc.LoaderTests): pass def test_unloadable(self): - with self.assertRaises(ImportError): - self.load_module('asdfjkl;') + name = 'asdfjkl;' + with self.assertRaises(ImportError) as cm: + self.load_module(name) + self.assertEqual(cm.exception.name, name) def test_main(): diff --git a/Lib/importlib/test/extension/test_path_hook.py b/Lib/importlib/test/extension/test_path_hook.py index 4610420d293..129e6e2975c 100644 --- a/Lib/importlib/test/extension/test_path_hook.py +++ b/Lib/importlib/test/extension/test_path_hook.py @@ -14,7 +14,8 @@ class PathHookTests(unittest.TestCase): # XXX Should it only work for directories containing an extension module? def hook(self, entry): - return _bootstrap._file_path_hook(entry) + return _bootstrap.FileFinder.path_hook((_bootstrap.ExtensionFileLoader, + imp.extension_suffixes(), False))(entry) def test_success(self): # Path hook should handle a directory where a known extension module diff --git a/Lib/importlib/test/extension/util.py b/Lib/importlib/test/extension/util.py index d1491697483..a266dd98c83 100644 --- a/Lib/importlib/test/extension/util.py +++ b/Lib/importlib/test/extension/util.py @@ -1,4 +1,5 @@ import imp +from importlib import machinery import os import sys @@ -6,10 +7,9 @@ PATH = None EXT = None FILENAME = None NAME = '_testcapi' -_file_exts = [x[0] for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION] try: for PATH in sys.path: - for EXT in _file_exts: + for EXT in machinery.EXTENSION_SUFFIXES: FILENAME = NAME + EXT FILEPATH = os.path.join(PATH, FILENAME) if os.path.exists(os.path.join(PATH, FILENAME)): @@ -18,4 +18,3 @@ try: PATH = EXT = FILENAME = FILEPATH = None except StopIteration: pass -del _file_exts diff --git a/Lib/importlib/test/frozen/test_loader.py b/Lib/importlib/test/frozen/test_loader.py index b685ef57084..6819a09b871 100644 --- a/Lib/importlib/test/frozen/test_loader.py +++ b/Lib/importlib/test/frozen/test_loader.py @@ -10,38 +10,46 @@ class LoaderTests(abc.LoaderTests): def test_module(self): with util.uncache('__hello__'), captured_stdout() as stdout: module = machinery.FrozenImporter.load_module('__hello__') - check = {'__name__': '__hello__', '__file__': '<frozen>', - '__package__': '', '__loader__': machinery.FrozenImporter} + check = {'__name__': '__hello__', + '__package__': '', + '__loader__': machinery.FrozenImporter, + } for attr, value in check.items(): self.assertEqual(getattr(module, attr), value) self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) def test_package(self): with util.uncache('__phello__'), captured_stdout() as stdout: module = machinery.FrozenImporter.load_module('__phello__') - check = {'__name__': '__phello__', '__file__': '<frozen>', - '__package__': '__phello__', '__path__': ['__phello__'], - '__loader__': machinery.FrozenImporter} + check = {'__name__': '__phello__', + '__package__': '__phello__', + '__path__': ['__phello__'], + '__loader__': machinery.FrozenImporter, + } for attr, value in check.items(): attr_value = getattr(module, attr) self.assertEqual(attr_value, value, "for __phello__.%s, %r != %r" % (attr, attr_value, value)) self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) def test_lacking_parent(self): with util.uncache('__phello__', '__phello__.spam'), \ captured_stdout() as stdout: module = machinery.FrozenImporter.load_module('__phello__.spam') - check = {'__name__': '__phello__.spam', '__file__': '<frozen>', + check = {'__name__': '__phello__.spam', '__package__': '__phello__', - '__loader__': machinery.FrozenImporter} + '__loader__': machinery.FrozenImporter, + } for attr, value in check.items(): attr_value = getattr(module, attr) self.assertEqual(attr_value, value, "for __phello__.spam.%s, %r != %r" % (attr, attr_value, value)) self.assertEqual(stdout.getvalue(), 'Hello world!\n') + self.assertFalse(hasattr(module, '__file__')) def test_module_reuse(self): with util.uncache('__hello__'), captured_stdout() as stdout: @@ -51,14 +59,21 @@ class LoaderTests(abc.LoaderTests): self.assertEqual(stdout.getvalue(), 'Hello world!\nHello world!\n') + def test_module_repr(self): + with util.uncache('__hello__'), captured_stdout(): + module = machinery.FrozenImporter.load_module('__hello__') + self.assertEqual(repr(module), + "<module '__hello__' (frozen)>") + def test_state_after_failure(self): # No way to trigger an error in a frozen module. pass def test_unloadable(self): assert machinery.FrozenImporter.find_module('_not_real') is None - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: machinery.FrozenImporter.load_module('_not_real') + self.assertEqual(cm.exception.name, '_not_real') class InspectLoaderTests(unittest.TestCase): @@ -92,8 +107,9 @@ class InspectLoaderTests(unittest.TestCase): # Raise ImportError for modules that are not frozen. for meth_name in ('get_code', 'get_source', 'is_package'): method = getattr(machinery.FrozenImporter, meth_name) - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: method('importlib') + self.assertEqual(cm.exception.name, 'importlib') def test_main(): diff --git a/Lib/importlib/test/import_/test___package__.py b/Lib/importlib/test/import_/test___package__.py index 5056ae59cca..783cde17294 100644 --- a/Lib/importlib/test/import_/test___package__.py +++ b/Lib/importlib/test/import_/test___package__.py @@ -67,7 +67,7 @@ class Using__package__(unittest.TestCase): def test_bunk__package__(self): globals = {'__package__': 42} - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): import_util.import_('', globals, {}, ['relimport'], 1) diff --git a/Lib/importlib/test/import_/test_api.py b/Lib/importlib/test/import_/test_api.py index 9075d427597..2fa1f909547 100644 --- a/Lib/importlib/test/import_/test_api.py +++ b/Lib/importlib/test/import_/test_api.py @@ -12,6 +12,13 @@ class APITest(unittest.TestCase): with self.assertRaises(TypeError): util.import_(42) + def test_negative_level(self): + # Raise ValueError when a negative level is specified. + # PEP 328 did away with sys.module None entries and the ambiguity of + # absolute/relative imports. + with self.assertRaises(ValueError): + util.import_('os', globals(), level=-1) + def test_main(): from test.support import run_unittest diff --git a/Lib/importlib/test/import_/test_caching.py b/Lib/importlib/test/import_/test_caching.py index 48dc64311af..3baff5501ed 100644 --- a/Lib/importlib/test/import_/test_caching.py +++ b/Lib/importlib/test/import_/test_caching.py @@ -34,8 +34,9 @@ class UseCache(unittest.TestCase): name = 'using_None' with util.uncache(name): sys.modules[name] = None - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: import_util.import_(name) + self.assertEqual(cm.exception.name, name) def create_mock(self, *names, return_=None): mock = util.mock_modules(*names) diff --git a/Lib/importlib/test/import_/test_fromlist.py b/Lib/importlib/test/import_/test_fromlist.py index b903e8e6112..4ff5f5e8254 100644 --- a/Lib/importlib/test/import_/test_fromlist.py +++ b/Lib/importlib/test/import_/test_fromlist.py @@ -1,6 +1,7 @@ """Test that the semantics relating to the 'fromlist' argument are correct.""" from .. import util from . import util as import_util +import imp import unittest class ReturnValue(unittest.TestCase): @@ -73,7 +74,8 @@ class HandlingFromlist(unittest.TestCase): def test_no_module_from_package(self): # [no module] with util.mock_modules('pkg.__init__') as importer: - with util.import_state(meta_path=[importer]): + with util.import_state(meta_path=[importer], + path_hooks=[imp.NullImporter]): module = import_util.import_('pkg', fromlist='non_existent') self.assertEqual(module.__name__, 'pkg') self.assertTrue(not hasattr(module, 'non_existent')) diff --git a/Lib/importlib/test/import_/test_meta_path.py b/Lib/importlib/test/import_/test_meta_path.py index 3b130c9a13c..2f65af99a9f 100644 --- a/Lib/importlib/test/import_/test_meta_path.py +++ b/Lib/importlib/test/import_/test_meta_path.py @@ -1,7 +1,10 @@ from .. import util from . import util as import_util +import importlib._bootstrap +import sys from types import MethodType import unittest +import warnings class CallingOrder(unittest.TestCase): @@ -33,6 +36,21 @@ class CallingOrder(unittest.TestCase): with util.import_state(meta_path=[first, second]): self.assertEqual(import_util.import_(mod_name), 42) + def test_empty(self): + # Raise an ImportWarning if sys.meta_path is empty. + module_name = 'nothing' + try: + del sys.modules[module_name] + except KeyError: + pass + with util.import_state(meta_path=[]): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertIsNone(importlib._bootstrap._find_module('nothing', + None)) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[-1].category, ImportWarning)) + class CallSignature(unittest.TestCase): diff --git a/Lib/importlib/test/import_/test_packages.py b/Lib/importlib/test/import_/test_packages.py index faadc32172b..931494d6f7c 100644 --- a/Lib/importlib/test/import_/test_packages.py +++ b/Lib/importlib/test/import_/test_packages.py @@ -3,6 +3,7 @@ from . import util as import_util import sys import unittest import importlib +from test import support class ParentModuleTests(unittest.TestCase): @@ -18,14 +19,88 @@ class ParentModuleTests(unittest.TestCase): def test_bad_parent(self): with util.mock_modules('pkg.module') as mock: with util.import_state(meta_path=[mock]): - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: import_util.import_('pkg.module') + self.assertEqual(cm.exception.name, 'pkg') + + def test_raising_parent_after_importing_child(self): + def __init__(): + import pkg.module + 1/0 + mock = util.mock_modules('pkg.__init__', 'pkg.module', + module_code={'pkg': __init__}) + with mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises(ZeroDivisionError): + import_util.import_('pkg') + self.assertFalse('pkg' in sys.modules) + self.assertTrue('pkg.module' in sys.modules) + with self.assertRaises(ZeroDivisionError): + import_util.import_('pkg.module') + self.assertFalse('pkg' in sys.modules) + self.assertTrue('pkg.module' in sys.modules) + + def test_raising_parent_after_relative_importing_child(self): + def __init__(): + from . import module + 1/0 + mock = util.mock_modules('pkg.__init__', 'pkg.module', + module_code={'pkg': __init__}) + with mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises((ZeroDivisionError, ImportError)): + # This raises ImportError on the "from . import module" + # line, not sure why. + import_util.import_('pkg') + self.assertFalse('pkg' in sys.modules) + with self.assertRaises((ZeroDivisionError, ImportError)): + import_util.import_('pkg.module') + self.assertFalse('pkg' in sys.modules) + # XXX False + #self.assertTrue('pkg.module' in sys.modules) + + def test_raising_parent_after_double_relative_importing_child(self): + def __init__(): + from ..subpkg import module + 1/0 + mock = util.mock_modules('pkg.__init__', 'pkg.subpkg.__init__', + 'pkg.subpkg.module', + module_code={'pkg.subpkg': __init__}) + with mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises((ZeroDivisionError, ImportError)): + # This raises ImportError on the "from ..subpkg import module" + # line, not sure why. + import_util.import_('pkg.subpkg') + self.assertFalse('pkg.subpkg' in sys.modules) + with self.assertRaises((ZeroDivisionError, ImportError)): + import_util.import_('pkg.subpkg.module') + self.assertFalse('pkg.subpkg' in sys.modules) + # XXX False + #self.assertTrue('pkg.subpkg.module' in sys.modules) def test_module_not_package(self): # Try to import a submodule from a non-package should raise ImportError. assert not hasattr(sys, '__path__') - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: import_util.import_('sys.no_submodules_here') + self.assertEqual(cm.exception.name, 'sys.no_submodules_here') + + def test_module_not_package_but_side_effects(self): + # If a module injects something into sys.modules as a side-effect, then + # pick up on that fact. + name = 'mod' + subname = name + '.b' + def module_injection(): + sys.modules[subname] = 'total bunk' + mock_modules = util.mock_modules('mod', + module_code={'mod': module_injection}) + with mock_modules as mock: + with util.import_state(meta_path=[mock]): + try: + submodule = import_util.import_(subname) + finally: + support.unload(subname) def test_main(): diff --git a/Lib/importlib/test/import_/test_path.py b/Lib/importlib/test/import_/test_path.py index 2faa23174b3..723f5b57a81 100644 --- a/Lib/importlib/test/import_/test_path.py +++ b/Lib/importlib/test/import_/test_path.py @@ -9,6 +9,7 @@ import tempfile from test import support from types import MethodType import unittest +import warnings class FinderTests(unittest.TestCase): @@ -42,6 +43,15 @@ class FinderTests(unittest.TestCase): loader = machinery.PathFinder.find_module(module, [path]) self.assertTrue(loader is importer) + def test_empty_list(self): + # An empty list should not count as asking for sys.path. + module = 'module' + path = '<test path>' + importer = util.mock_modules(module) + with util.import_state(path_importer_cache={path: importer}, + path=[path]): + self.assertIsNone(machinery.PathFinder.find_module('module', [])) + def test_path_hooks(self): # Test that sys.path_hooks is used. # Test that sys.path_importer_cache is set. @@ -55,77 +65,34 @@ class FinderTests(unittest.TestCase): self.assertTrue(path in sys.path_importer_cache) self.assertTrue(sys.path_importer_cache[path] is importer) - def test_path_importer_cache_has_None(self): - # Test that if sys.path_importer_cache has None that None is returned. - clear_cache = {path: None for path in sys.path} - with util.import_state(path_importer_cache=clear_cache): - for name in ('asynchat', 'sys', '<test module>'): - self.assertTrue(machinery.PathFinder.find_module(name) is None) - - def test_path_importer_cache_has_None_continues(self): - # Test that having None in sys.path_importer_cache causes the search to - # continue. - path = '<test path>' + def test_empty_path_hooks(self): + # Test that if sys.path_hooks is empty a warning is raised, + # sys.path_importer_cache gets None set, and PathFinder returns None. + path_entry = 'bogus_path' + with util.import_state(path_importer_cache={}, path_hooks=[], + path=[path_entry]): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertIsNone(machinery.PathFinder.find_module('os')) + self.assertIsNone(sys.path_importer_cache[path_entry]) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[-1].category, ImportWarning)) + + def test_path_importer_cache_empty_string(self): + # The empty string should create a finder using the cwd. + path = '' module = '<test module>' importer = util.mock_modules(module) - with util.import_state(path=['1', '2'], - path_importer_cache={'1': None, '2': importer}): + hook = import_util.mock_path_hook(os.curdir, importer=importer) + with util.import_state(path=[path], path_hooks=[hook]): loader = machinery.PathFinder.find_module(module) - self.assertTrue(loader is importer) - - - -class DefaultPathFinderTests(unittest.TestCase): - - """Test importlib._bootstrap._DefaultPathFinder.""" - - def test_implicit_hooks(self): - # Test that the implicit path hooks are used. - bad_path = '<path>' - module = '<module>' - assert not os.path.exists(bad_path) - existing_path = tempfile.mkdtemp() - try: - with util.import_state(): - nothing = _bootstrap._DefaultPathFinder.find_module(module, - path=[existing_path]) - self.assertTrue(nothing is None) - self.assertTrue(existing_path in sys.path_importer_cache) - result = isinstance(sys.path_importer_cache[existing_path], - imp.NullImporter) - self.assertFalse(result) - nothing = _bootstrap._DefaultPathFinder.find_module(module, - path=[bad_path]) - self.assertTrue(nothing is None) - self.assertTrue(bad_path in sys.path_importer_cache) - self.assertTrue(isinstance(sys.path_importer_cache[bad_path], - imp.NullImporter)) - finally: - os.rmdir(existing_path) - - - def test_path_importer_cache_has_None(self): - # Test that the default hook is used when sys.path_importer_cache - # contains None for a path. - module = '<test module>' - importer = util.mock_modules(module) - path = '<test path>' - # XXX Not blackbox. - original_hook = _bootstrap._DEFAULT_PATH_HOOK - mock_hook = import_util.mock_path_hook(path, importer=importer) - _bootstrap._DEFAULT_PATH_HOOK = mock_hook - try: - with util.import_state(path_importer_cache={path: None}): - loader = _bootstrap._DefaultPathFinder.find_module(module, - path=[path]) - self.assertTrue(loader is importer) - finally: - _bootstrap._DEFAULT_PATH_HOOK = original_hook + self.assertIs(loader, importer) + self.assertIn(os.curdir, sys.path_importer_cache) def test_main(): from test.support import run_unittest - run_unittest(FinderTests, DefaultPathFinderTests) + run_unittest(FinderTests) if __name__ == '__main__': test_main() diff --git a/Lib/importlib/test/import_/test_relative_imports.py b/Lib/importlib/test/import_/test_relative_imports.py index a0f6b2d827d..4569c26424f 100644 --- a/Lib/importlib/test/import_/test_relative_imports.py +++ b/Lib/importlib/test/import_/test_relative_imports.py @@ -193,6 +193,20 @@ class RelativeImports(unittest.TestCase): self.assertEqual(module.__name__, '__runpy_pkg__.uncle.cousin') self.relative_import_test(create, globals_, callback) + def test_import_relative_import_no_fromlist(self): + # Import a relative module w/ no fromlist. + create = ['crash.__init__', 'crash.mod'] + globals_ = [{'__package__': 'crash', '__name__': 'crash'}] + def callback(global_): + import_util.import_('crash') + mod = import_util.import_('mod', global_, {}, [], 1) + self.assertEqual(mod.__name__, 'crash.mod') + self.relative_import_test(create, globals_, callback) + + def test_relative_import_no_globals(self): + # No globals for a relative import is an error. + with self.assertRaises(KeyError): + import_util.import_('sys', level=1) def test_main(): diff --git a/Lib/importlib/test/import_/util.py b/Lib/importlib/test/import_/util.py index 649c5ed27c3..86ac065e64c 100644 --- a/Lib/importlib/test/import_/util.py +++ b/Lib/importlib/test/import_/util.py @@ -1,6 +1,5 @@ import functools import importlib -import importlib._bootstrap import unittest diff --git a/Lib/importlib/test/regrtest.py b/Lib/importlib/test/regrtest.py index b103ae7d0e9..a5be11fd4ee 100644 --- a/Lib/importlib/test/regrtest.py +++ b/Lib/importlib/test/regrtest.py @@ -5,13 +5,6 @@ invalidates are automatically skipped if the entire test suite is run. Otherwise all command-line options valid for test.regrtest are also valid for this script. -XXX FAILING - * test_import - - test_incorrect_code_name - file name differing between __file__ and co_filename (r68360 on trunk) - - test_import_by_filename - exception for trying to import by file name does not match - """ import importlib import sys @@ -19,17 +12,6 @@ from test import regrtest if __name__ == '__main__': __builtins__.__import__ = importlib.__import__ - - exclude = ['--exclude', - 'test_frozen', # Does not expect __loader__ attribute - 'test_pkg', # Does not expect __loader__ attribute - 'test_pydoc', # Does not expect __loader__ attribute - ] - - # Switching on --exclude implies running all test but the ones listed, so - # only use it when one is not running an explicit test - if len(sys.argv) == 1: - # No programmatic way to specify tests to exclude - sys.argv.extend(exclude) + sys.path_importer_cache.clear() regrtest.main(quiet=True, verbose2=True) diff --git a/Lib/importlib/test/source/test_abc_loader.py b/Lib/importlib/test/source/test_abc_loader.py index 32459074a07..fc98b93b82b 100644 --- a/Lib/importlib/test/source/test_abc_loader.py +++ b/Lib/importlib/test/source/test_abc_loader.py @@ -40,8 +40,10 @@ class SourceLoaderMock(SourceOnlyLoaderMock): def __init__(self, path, magic=imp.get_magic()): super().__init__(path) self.bytecode_path = imp.cache_from_source(self.path) + self.source_size = len(self.source) data = bytearray(magic) - data.extend(marshal._w_long(self.source_mtime)) + data.extend(importlib._w_long(self.source_mtime)) + data.extend(importlib._w_long(self.source_size)) code_object = compile(self.source, self.path, 'exec', dont_inherit=True) data.extend(marshal.dumps(code_object)) @@ -56,9 +58,9 @@ class SourceLoaderMock(SourceOnlyLoaderMock): else: raise IOError - def path_mtime(self, path): + def path_stats(self, path): assert path == self.path - return self.source_mtime + return {'mtime': self.source_mtime, 'size': self.source_size} def set_data(self, path, data): self.written[path] = bytes(data) @@ -102,7 +104,7 @@ class PyLoaderMock(abc.PyLoader): warnings.simplefilter("always") path = super().get_filename(name) assert len(w) == 1 - assert issubclass(w[0].category, PendingDeprecationWarning) + assert issubclass(w[0].category, DeprecationWarning) return path @@ -198,7 +200,7 @@ class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): warnings.simplefilter("always") code_object = super().get_code(name) assert len(w) == 1 - assert issubclass(w[0].category, PendingDeprecationWarning) + assert issubclass(w[0].category, DeprecationWarning) return code_object class PyLoaderTests(testing_abc.LoaderTests): @@ -469,8 +471,9 @@ class BadBytecodeFailureTests(unittest.TestCase): {'path': os.path.join('path', 'to', 'mod'), 'magic': bad_magic}} mock = PyPycLoaderMock({name: None}, bc) - with util.uncache(name), self.assertRaises(ImportError): + with util.uncache(name), self.assertRaises(ImportError) as cm: mock.load_module(name) + self.assertEqual(cm.exception.name, name) def test_no_bytecode(self): # Missing code object bytecode should lead to an EOFError. @@ -514,8 +517,9 @@ class MissingPathsTests(unittest.TestCase): # If all *_path methods return None, raise ImportError. name = 'mod' mock = PyPycLoaderMock({name: None}) - with util.uncache(name), self.assertRaises(ImportError): + with util.uncache(name), self.assertRaises(ImportError) as cm: mock.load_module(name) + self.assertEqual(cm.exception.name, name) def test_source_path_ImportError(self): # An ImportError from source_path should trigger an ImportError. @@ -531,7 +535,7 @@ class MissingPathsTests(unittest.TestCase): mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}) bad_meth = types.MethodType(raise_ImportError, mock) mock.bytecode_path = bad_meth - with util.uncache(name), self.assertRaises(ImportError): + with util.uncache(name), self.assertRaises(ImportError) as cm: mock.load_module(name) @@ -592,15 +596,17 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness): def raise_IOError(path): raise IOError self.loader.get_data = raise_IOError - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: self.loader.get_source(self.name) + self.assertEqual(cm.exception.name, self.name) def test_is_package(self): # Properly detect when loading a package. - self.setUp(is_package=True) - self.assertTrue(self.loader.is_package(self.name)) self.setUp(is_package=False) self.assertFalse(self.loader.is_package(self.name)) + self.setUp(is_package=True) + self.assertTrue(self.loader.is_package(self.name)) + self.assertFalse(self.loader.is_package(self.name + '.__init__')) def test_get_code(self): # Verify the code object is created. @@ -656,7 +662,8 @@ class SourceLoaderBytecodeTests(SourceLoaderTestHarness): if bytecode_written: self.assertIn(self.cached, self.loader.written) data = bytearray(imp.get_magic()) - data.extend(marshal._w_long(self.loader.source_mtime)) + data.extend(importlib._w_long(self.loader.source_mtime)) + data.extend(importlib._w_long(self.loader.source_size)) data.extend(marshal.dumps(code_object)) self.assertEqual(self.loader.written[self.cached], bytes(data)) @@ -847,7 +854,7 @@ class AbstractMethodImplTests(unittest.TestCase): # Required abstractmethods. self.raises_NotImplementedError(ins, 'get_filename', 'get_data') # Optional abstractmethods. - self.raises_NotImplementedError(ins,'path_mtime', 'set_data') + self.raises_NotImplementedError(ins,'path_stats', 'set_data') def test_PyLoader(self): self.raises_NotImplementedError(self.PyLoader(), 'source_path', diff --git a/Lib/importlib/test/source/test_case_sensitivity.py b/Lib/importlib/test/source/test_case_sensitivity.py index 73777de4ba4..21a4378ce9d 100644 --- a/Lib/importlib/test/source/test_case_sensitivity.py +++ b/Lib/importlib/test/source/test_case_sensitivity.py @@ -1,7 +1,9 @@ """Test case-sensitivity (PEP 235).""" from importlib import _bootstrap +from importlib import machinery from .. import util from . import util as source_util +import imp import os import sys from test import support as test_support @@ -19,9 +21,13 @@ class CaseSensitivityTest(unittest.TestCase): assert name != name.lower() def find(self, path): - finder = _bootstrap._FileFinder(path, - _bootstrap._SourceFinderDetails(), - _bootstrap._SourcelessFinderDetails()) + finder = machinery.FileFinder(path, + (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, + True), + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES, + True)) return finder.find_module(self.name) def sensitivity_test(self): @@ -37,6 +43,9 @@ class CaseSensitivityTest(unittest.TestCase): def test_sensitive(self): with test_support.EnvironmentVarGuard() as env: env.unset('PYTHONCASEOK') + if b'PYTHONCASEOK' in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') sensitive, insensitive = self.sensitivity_test() self.assertTrue(hasattr(sensitive, 'load_module')) self.assertIn(self.name, sensitive.get_filename(self.name)) @@ -45,6 +54,9 @@ class CaseSensitivityTest(unittest.TestCase): def test_insensitive(self): with test_support.EnvironmentVarGuard() as env: env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') sensitive, insensitive = self.sensitivity_test() self.assertTrue(hasattr(sensitive, 'load_module')) self.assertIn(self.name, sensitive.get_filename(self.name)) diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py index c7a7d8fbcae..ffa0c24e091 100644 --- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -1,5 +1,6 @@ +from ... import _bootstrap import importlib -from importlib import _bootstrap +import importlib.abc from .. import abc from .. import util from . import util as source_util @@ -24,10 +25,44 @@ class SimpleTest(unittest.TestCase): """ + def test_load_module_API(self): + # If fullname is not specified that assume self.name is desired. + class TesterMixin(importlib.abc.Loader): + def load_module(self, fullname): return fullname + + class Tester(importlib.abc.FileLoader, TesterMixin): + def get_code(self, _): pass + def get_source(self, _): pass + def is_package(self, _): pass + + name = 'mod_name' + loader = Tester(name, 'some_path') + self.assertEqual(name, loader.load_module()) + self.assertEqual(name, loader.load_module(None)) + self.assertEqual(name, loader.load_module(name)) + with self.assertRaises(ImportError): + loader.load_module(loader.name + 'XXX') + + def test_get_filename_API(self): + # If fullname is not set then assume self.path is desired. + class Tester(importlib.abc.FileLoader): + def get_code(self, _): pass + def get_source(self, _): pass + def is_package(self, _): pass + + path = 'some_path' + name = 'some_name' + loader = Tester(name, path) + self.assertEqual(path, loader.get_filename(name)) + self.assertEqual(path, loader.get_filename()) + self.assertEqual(path, loader.get_filename(None)) + with self.assertRaises(ImportError): + loader.get_filename(name + 'XXX') + # [basic] def test_module(self): with source_util.create_modules('_temp') as mapping: - loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + loader = _bootstrap.SourceFileLoader('_temp', mapping['_temp']) module = loader.load_module('_temp') self.assertTrue('_temp' in sys.modules) check = {'__name__': '_temp', '__file__': mapping['_temp'], @@ -37,7 +72,7 @@ class SimpleTest(unittest.TestCase): def test_package(self): with source_util.create_modules('_pkg.__init__') as mapping: - loader = _bootstrap._SourceFileLoader('_pkg', + loader = _bootstrap.SourceFileLoader('_pkg', mapping['_pkg.__init__']) module = loader.load_module('_pkg') self.assertTrue('_pkg' in sys.modules) @@ -50,7 +85,7 @@ class SimpleTest(unittest.TestCase): def test_lacking_parent(self): with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: - loader = _bootstrap._SourceFileLoader('_pkg.mod', + loader = _bootstrap.SourceFileLoader('_pkg.mod', mapping['_pkg.mod']) module = loader.load_module('_pkg.mod') self.assertTrue('_pkg.mod' in sys.modules) @@ -65,17 +100,12 @@ class SimpleTest(unittest.TestCase): def test_module_reuse(self): with source_util.create_modules('_temp') as mapping: - loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + loader = _bootstrap.SourceFileLoader('_temp', mapping['_temp']) module = loader.load_module('_temp') module_id = id(module) module_dict_id = id(module.__dict__) with open(mapping['_temp'], 'w') as file: file.write("testing_var = 42\n") - # For filesystems where the mtime is only to a second granularity, - # everything that has happened above can be too fast; - # force an mtime on the source that is guaranteed to be different - # than the original mtime. - loader.path_mtime = self.fake_mtime(loader.path_mtime) module = loader.load_module('_temp') self.assertTrue('testing_var' in module.__dict__, "'testing_var' not in " @@ -95,7 +125,7 @@ class SimpleTest(unittest.TestCase): setattr(orig_module, attr, value) with open(mapping[name], 'w') as file: file.write('+++ bad syntax +++') - loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + loader = _bootstrap.SourceFileLoader('_temp', mapping['_temp']) with self.assertRaises(SyntaxError): loader.load_module(name) for attr in attributes: @@ -106,7 +136,7 @@ class SimpleTest(unittest.TestCase): with source_util.create_modules('_temp') as mapping: with open(mapping['_temp'], 'w') as file: file.write('=') - loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + loader = _bootstrap.SourceFileLoader('_temp', mapping['_temp']) with self.assertRaises(SyntaxError): loader.load_module('_temp') self.assertTrue('_temp' not in sys.modules) @@ -119,7 +149,7 @@ class SimpleTest(unittest.TestCase): file.write("# test file for importlib") try: with util.uncache('_temp'): - loader = _bootstrap._SourceFileLoader('_temp', file_path) + loader = _bootstrap.SourceFileLoader('_temp', file_path) mod = loader.load_module('_temp') self.assertEqual(file_path, mod.__file__) self.assertEqual(imp.cache_from_source(file_path), @@ -145,7 +175,7 @@ class SimpleTest(unittest.TestCase): if e.errno != getattr(errno, 'EOVERFLOW', None): raise self.skipTest("cannot set modification time to large integer ({})".format(e)) - loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + loader = _bootstrap.SourceFileLoader('_temp', mapping['_temp']) mod = loader.load_module('_temp') # Sanity checks. self.assertEqual(mod.__cached__, compiled) @@ -215,10 +245,17 @@ class BadBytecodeTest(unittest.TestCase): del_source=del_source) test('_temp', mapping, bc_path) + def _test_partial_size(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:11], + del_source=del_source) + test('_temp', mapping, bc_path) + def _test_no_marshal(self, *, del_source=False): with source_util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8], + lambda bc: bc[:12], del_source=del_source) file_path = mapping['_temp'] if not del_source else bc_path with self.assertRaises(EOFError): @@ -227,16 +264,18 @@ class BadBytecodeTest(unittest.TestCase): def _test_non_code_marshal(self, *, del_source=False): with source_util.create_modules('_temp') as mapping: bytecode_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8] + marshal.dumps(b'abcd'), + lambda bc: bc[:12] + marshal.dumps(b'abcd'), del_source=del_source) file_path = mapping['_temp'] if not del_source else bytecode_path - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: self.import_(file_path, '_temp') + self.assertEqual(cm.exception.name, '_temp') + self.assertEqual(cm.exception.path, bytecode_path) def _test_bad_marshal(self, *, del_source=False): with source_util.create_modules('_temp') as mapping: bytecode_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8] + b'<test>', + lambda bc: bc[:12] + b'<test>', del_source=del_source) file_path = mapping['_temp'] if not del_source else bytecode_path with self.assertRaises(EOFError): @@ -251,7 +290,7 @@ class BadBytecodeTest(unittest.TestCase): class SourceLoaderBadBytecodeTest(BadBytecodeTest): - loader = _bootstrap._SourceFileLoader + loader = _bootstrap.SourceFileLoader @source_util.writes_bytecode_files def test_empty_file(self): @@ -260,7 +299,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bytecode_path): self.import_(mapping[name], name) with open(bytecode_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_empty_file(test) @@ -268,7 +307,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bytecode_path): self.import_(mapping[name], name) with open(bytecode_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_partial_magic(test) @@ -279,7 +318,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bytecode_path): self.import_(mapping[name], name) with open(bytecode_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_magic_only(test) @@ -301,11 +340,22 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bc_path): self.import_(mapping[name], name) with open(bc_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_partial_timestamp(test) @source_util.writes_bytecode_files + def test_partial_size(self): + # When the size is partial, regenerate the .pyc, else + # raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 12) + + self._test_partial_size(test) + + @source_util.writes_bytecode_files def test_no_marshal(self): # When there is only the magic number and timestamp, raise EOFError. self._test_no_marshal() @@ -364,19 +414,23 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): - loader = _bootstrap._SourcelessFileLoader + loader = _bootstrap.SourcelessFileLoader def test_empty_file(self): def test(name, mapping, bytecode_path): - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: self.import_(bytecode_path, name) + self.assertEqual(cm.exception.name, name) + self.assertEqual(cm.exception.path, bytecode_path) self._test_empty_file(test, del_source=True) def test_partial_magic(self): def test(name, mapping, bytecode_path): - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: self.import_(bytecode_path, name) + self.assertEqual(cm.exception.name, name) + self.assertEqual(cm.exception.path, bytecode_path) self._test_partial_magic(test, del_source=True) def test_magic_only(self): @@ -388,8 +442,10 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): def test_bad_magic(self): def test(name, mapping, bytecode_path): - with self.assertRaises(ImportError): + with self.assertRaises(ImportError) as cm: self.import_(bytecode_path, name) + self.assertEqual(cm.exception.name, name) + self.assertEqual(cm.exception.path, bytecode_path) self._test_bad_magic(test, del_source=True) @@ -400,6 +456,13 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): self._test_partial_timestamp(test, del_source=True) + def test_partial_size(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_size(test, del_source=True) + def test_no_marshal(self): self._test_no_marshal(del_source=True) diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py index 7b9088da0ce..b22c1030442 100644 --- a/Lib/importlib/test/source/test_finder.py +++ b/Lib/importlib/test/source/test_finder.py @@ -1,10 +1,12 @@ -from importlib import _bootstrap from .. import abc from . import util as source_util -from test.support import make_legacy_pyc -import os + +from importlib import machinery import errno +import imp +import os import py_compile +from test.support import make_legacy_pyc import unittest import warnings @@ -34,9 +36,11 @@ class FinderTests(abc.FinderTests): """ def import_(self, root, module): - finder = _bootstrap._FileFinder(root, - _bootstrap._SourceFinderDetails(), - _bootstrap._SourcelessFinderDetails()) + loader_details = [(machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True), + (machinery.SourcelessFileLoader, + machinery.BYTECODE_SUFFIXES, True)] + finder = machinery.FileFinder(root, *loader_details) return finder.find_module(module) def run_test(self, test, create=None, *, compile_=None, unlink=None): @@ -102,39 +106,21 @@ class FinderTests(abc.FinderTests): loader = self.import_(pkg_dir, 'pkg.sub') self.assertTrue(hasattr(loader, 'load_module')) - # [sub empty] - def test_empty_sub_directory(self): - context = source_util.create_modules('pkg.__init__', 'pkg.sub.__init__') - with warnings.catch_warnings(): - warnings.simplefilter("error", ImportWarning) - with context as mapping: - os.unlink(mapping['pkg.sub.__init__']) - pkg_dir = os.path.dirname(mapping['pkg.__init__']) - with self.assertRaises(ImportWarning): - self.import_(pkg_dir, 'pkg.sub') - # [package over modules] def test_package_over_module(self): name = '_temp' loader = self.run_test(name, {'{0}.__init__'.format(name), name}) - self.assertTrue('__init__' in loader.get_filename(name)) - + self.assertIn('__init__', loader.get_filename(name)) def test_failure(self): with source_util.create_modules('blah') as mapping: nothing = self.import_(mapping['.root'], 'sdfsadsadf') self.assertTrue(nothing is None) - # [empty dir] - def test_empty_dir(self): - with warnings.catch_warnings(): - warnings.simplefilter("error", ImportWarning) - with self.assertRaises(ImportWarning): - self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'}) - def test_empty_string_for_dir(self): # The empty string from sys.path means to search in the cwd. - finder = _bootstrap._FileFinder('', _bootstrap._SourceFinderDetails()) + finder = machinery.FileFinder('', (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) with open('mod.py', 'w') as file: file.write("# test file for importlib") try: @@ -143,6 +129,14 @@ class FinderTests(abc.FinderTests): finally: os.unlink('mod.py') + def test_invalidate_caches(self): + # invalidate_caches() should reset the mtime. + finder = machinery.FileFinder('', (machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) + finder._path_mtime = 42 + finder.invalidate_caches() + self.assertEqual(finder._path_mtime, -1) + def test_main(): from test.support import run_unittest diff --git a/Lib/importlib/test/source/test_path_hook.py b/Lib/importlib/test/source/test_path_hook.py index 374f7b6ad3e..54c06995014 100644 --- a/Lib/importlib/test/source/test_path_hook.py +++ b/Lib/importlib/test/source/test_path_hook.py @@ -1,5 +1,7 @@ -from importlib import _bootstrap from . import util as source_util + +from importlib import machinery +import imp import unittest @@ -7,14 +9,18 @@ class PathHookTest(unittest.TestCase): """Test the path hook for source.""" + def path_hook(self): + return machinery.FileFinder.path_hook((machinery.SourceFileLoader, + machinery.SOURCE_SUFFIXES, True)) + def test_success(self): with source_util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']), + self.assertTrue(hasattr(self.path_hook()(mapping['.root']), 'find_module')) def test_empty_string(self): # The empty string represents the cwd. - self.assertTrue(hasattr(_bootstrap._file_path_hook(''), 'find_module')) + self.assertTrue(hasattr(self.path_hook()(''), 'find_module')) def test_main(): diff --git a/Lib/importlib/test/source/test_source_encoding.py b/Lib/importlib/test/source/test_source_encoding.py index 794a3df2463..0ca51954390 100644 --- a/Lib/importlib/test/source/test_source_encoding.py +++ b/Lib/importlib/test/source/test_source_encoding.py @@ -1,6 +1,6 @@ -from importlib import _bootstrap from . import util as source_util +from importlib import _bootstrap import codecs import re import sys @@ -35,8 +35,8 @@ class EncodingTest(unittest.TestCase): with source_util.create_modules(self.module_name) as mapping: with open(mapping[self.module_name], 'wb') as file: file.write(source) - loader = _bootstrap._SourceFileLoader(self.module_name, - mapping[self.module_name]) + loader = _bootstrap.SourceFileLoader(self.module_name, + mapping[self.module_name]) return loader.load_module(self.module_name) def create_source(self, encoding): @@ -97,7 +97,7 @@ class LineEndingTest(unittest.TestCase): with source_util.create_modules(module_name) as mapping: with open(mapping[module_name], 'wb') as file: file.write(source) - loader = _bootstrap._SourceFileLoader(module_name, + loader = _bootstrap.SourceFileLoader(module_name, mapping[module_name]) return loader.load_module(module_name) diff --git a/Lib/importlib/test/test_abc.py b/Lib/importlib/test/test_abc.py index 0ecbe390ad6..008bd21ff88 100644 --- a/Lib/importlib/test/test_abc.py +++ b/Lib/importlib/test/test_abc.py @@ -50,7 +50,7 @@ class InspectLoader(InheritanceTests, unittest.TestCase): superclasses = [abc.Loader] subclasses = [abc.PyLoader, machinery.BuiltinImporter, - machinery.FrozenImporter] + machinery.FrozenImporter, machinery.ExtensionFileLoader] class ExecutionLoader(InheritanceTests, unittest.TestCase): @@ -59,9 +59,16 @@ class ExecutionLoader(InheritanceTests, unittest.TestCase): subclasses = [abc.PyLoader] +class FileLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.ResourceLoader, abc.ExecutionLoader] + subclasses = [machinery.SourceFileLoader, machinery.SourcelessFileLoader] + + class SourceLoader(InheritanceTests, unittest.TestCase): superclasses = [abc.ResourceLoader, abc.ExecutionLoader] + subclasses = [machinery.SourceFileLoader] class PyLoader(InheritanceTests, unittest.TestCase): diff --git a/Lib/importlib/test/test_api.py b/Lib/importlib/test/test_api.py index a151626de7d..b7d6cb4effe 100644 --- a/Lib/importlib/test/test_api.py +++ b/Lib/importlib/test/test_api.py @@ -84,9 +84,85 @@ class ImportModuleTests(unittest.TestCase): importlib.import_module('a.b') self.assertEqual(b_load_count, 1) + +class FindLoaderTests(unittest.TestCase): + + class FakeMetaFinder: + @staticmethod + def find_module(name, path=None): return name, path + + def test_sys_modules(self): + # If a module with __loader__ is in sys.modules, then return it. + name = 'some_mod' + with util.uncache(name): + module = imp.new_module(name) + loader = 'a loader!' + module.__loader__ = loader + sys.modules[name] = module + found = importlib.find_loader(name) + self.assertEqual(loader, found) + + def test_sys_modules_loader_is_None(self): + # If sys.modules[name].__loader__ is None, raise ValueError. + name = 'some_mod' + with util.uncache(name): + module = imp.new_module(name) + module.__loader__ = None + sys.modules[name] = module + with self.assertRaises(ValueError): + importlib.find_loader(name) + + def test_success(self): + # Return the loader found on sys.meta_path. + name = 'some_mod' + with util.uncache(name): + with util.import_state(meta_path=[self.FakeMetaFinder]): + self.assertEqual((name, None), importlib.find_loader(name)) + + def test_success_path(self): + # Searching on a path should work. + name = 'some_mod' + path = 'path to some place' + with util.uncache(name): + with util.import_state(meta_path=[self.FakeMetaFinder]): + self.assertEqual((name, path), + importlib.find_loader(name, path)) + + def test_nothing(self): + # None is returned upon failure to find a loader. + self.assertIsNone(importlib.find_loader('nevergoingtofindthismodule')) + + +class InvalidateCacheTests(unittest.TestCase): + + def test_method_called(self): + # If defined the method should be called. + class InvalidatingNullFinder: + def __init__(self, *ignored): + self.called = False + def find_module(self, *args): + return None + def invalidate_caches(self): + self.called = True + + key = 'gobledeegook' + ins = InvalidatingNullFinder() + sys.path_importer_cache[key] = ins + self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key)) + importlib.invalidate_caches() + self.assertTrue(ins.called) + + def test_method_lacking(self): + # There should be no issues if the method is not defined. + key = 'gobbledeegook' + sys.path_importer_cache[key] = imp.NullImporter('abc') + self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key)) + importlib.invalidate_caches() # Shouldn't trigger an exception. + + def test_main(): from test.support import run_unittest - run_unittest(ImportModuleTests) + run_unittest(ImportModuleTests, FindLoaderTests, InvalidateCacheTests) if __name__ == '__main__': diff --git a/Lib/importlib/test/test_locks.py b/Lib/importlib/test/test_locks.py new file mode 100644 index 00000000000..7faff490ae5 --- /dev/null +++ b/Lib/importlib/test/test_locks.py @@ -0,0 +1,115 @@ +from importlib import _bootstrap +import time +import unittest +import weakref + +from test import support + +try: + import threading +except ImportError: + threading = None +else: + from test import lock_tests + + +LockType = _bootstrap._ModuleLock +DeadlockError = _bootstrap._DeadlockError + + +if threading is not None: + class ModuleLockAsRLockTests(lock_tests.RLockTests): + locktype = staticmethod(lambda: LockType("some_lock")) + + # _is_owned() unsupported + test__is_owned = None + # acquire(blocking=False) unsupported + test_try_acquire = None + test_try_acquire_contended = None + # `with` unsupported + test_with = None + # acquire(timeout=...) unsupported + test_timeout = None + # _release_save() unsupported + test_release_save_unacquired = None + +else: + class ModuleLockAsRLockTests(unittest.TestCase): + pass + + +@unittest.skipUnless(threading, "threads needed for this test") +class DeadlockAvoidanceTests(unittest.TestCase): + + def run_deadlock_avoidance_test(self, create_deadlock): + NLOCKS = 10 + locks = [LockType(str(i)) for i in range(NLOCKS)] + pairs = [(locks[i], locks[(i+1)%NLOCKS]) for i in range(NLOCKS)] + if create_deadlock: + NTHREADS = NLOCKS + else: + NTHREADS = NLOCKS - 1 + barrier = threading.Barrier(NTHREADS) + results = [] + def _acquire(lock): + """Try to acquire the lock. Return True on success, False on deadlock.""" + try: + lock.acquire() + except DeadlockError: + return False + else: + return True + def f(): + a, b = pairs.pop() + ra = _acquire(a) + barrier.wait() + rb = _acquire(b) + results.append((ra, rb)) + if rb: + b.release() + if ra: + a.release() + lock_tests.Bunch(f, NTHREADS).wait_for_finished() + self.assertEqual(len(results), NTHREADS) + return results + + def test_deadlock(self): + results = self.run_deadlock_avoidance_test(True) + # One of the threads detected a potential deadlock on its second + # acquire() call. + self.assertEqual(results.count((True, False)), 1) + self.assertEqual(results.count((True, True)), len(results) - 1) + + def test_no_deadlock(self): + results = self.run_deadlock_avoidance_test(False) + self.assertEqual(results.count((True, False)), 0) + self.assertEqual(results.count((True, True)), len(results)) + + +class LifetimeTests(unittest.TestCase): + + def test_lock_lifetime(self): + name = "xyzzy" + self.assertNotIn(name, _bootstrap._module_locks) + lock = _bootstrap._get_module_lock(name) + self.assertIn(name, _bootstrap._module_locks) + wr = weakref.ref(lock) + del lock + support.gc_collect() + self.assertNotIn(name, _bootstrap._module_locks) + self.assertIs(wr(), None) + + def test_all_locks(self): + support.gc_collect() + self.assertEqual(0, len(_bootstrap._module_locks), _bootstrap._module_locks) + + +@support.reap_threads +def test_main(): + support.run_unittest(ModuleLockAsRLockTests, + DeadlockAvoidanceTests, + LifetimeTests) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/importlib/test/test_util.py b/Lib/importlib/test/test_util.py index 602447f09e4..e477f171f0c 100644 --- a/Lib/importlib/test/test_util.py +++ b/Lib/importlib/test/test_util.py @@ -59,10 +59,57 @@ class ModuleForLoaderTests(unittest.TestCase): self.raise_exception(name) self.assertIs(module, sys.modules[name]) + def test_decorator_attrs(self): + def fxn(self, module): pass + wrapped = util.module_for_loader(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) + + def test_false_module(self): + # If for some odd reason a module is considered false, still return it + # from sys.modules. + class FalseModule(types.ModuleType): + def __bool__(self): return False + + name = 'mod' + module = FalseModule(name) + with test_util.uncache(name): + self.assertFalse(module) + sys.modules[name] = module + given = self.return_module(name) + self.assertTrue(given is module) + + def test_attributes_set(self): + # __name__, __loader__, and __package__ should be set (when + # is_package() is defined; undefined implicitly tested elsewhere). + class FakeLoader: + def __init__(self, is_package): + self._pkg = is_package + def is_package(self, name): + return self._pkg + @util.module_for_loader + def load_module(self, module): + return module + + name = 'pkg.mod' + with test_util.uncache(name): + loader = FakeLoader(False) + module = loader.load_module(name) + self.assertEqual(module.__name__, name) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__package__, 'pkg') -class SetPackageTests(unittest.TestCase): + name = 'pkg.sub' + with test_util.uncache(name): + loader = FakeLoader(True) + module = loader.load_module(name) + self.assertEqual(module.__name__, name) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__package__, name) +class SetPackageTests(unittest.TestCase): + """Tests for importlib.util.set_package.""" def verify(self, module, expect): @@ -108,10 +155,53 @@ class SetPackageTests(unittest.TestCase): module.__package__ = value self.verify(module, value) + def test_decorator_attrs(self): + def fxn(module): pass + wrapped = util.set_package(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) + + +class ResolveNameTests(unittest.TestCase): + + """Tests importlib.util.resolve_name().""" + + def test_absolute(self): + # bacon + self.assertEqual('bacon', util.resolve_name('bacon', None)) + + def test_aboslute_within_package(self): + # bacon in spam + self.assertEqual('bacon', util.resolve_name('bacon', 'spam')) + + def test_no_package(self): + # .bacon in '' + with self.assertRaises(ValueError): + util.resolve_name('.bacon', '') + + def test_in_package(self): + # .bacon in spam + self.assertEqual('spam.eggs.bacon', + util.resolve_name('.bacon', 'spam.eggs')) + + def test_other_package(self): + # ..bacon in spam.bacon + self.assertEqual('spam.bacon', + util.resolve_name('..bacon', 'spam.eggs')) + + def test_escape(self): + # ..bacon in spam + with self.assertRaises(ValueError): + util.resolve_name('..bacon', 'spam') + def test_main(): from test import support - support.run_unittest(ModuleForLoaderTests, SetPackageTests) + support.run_unittest( + ModuleForLoaderTests, + SetPackageTests, + ResolveNameTests + ) if __name__ == '__main__': diff --git a/Lib/importlib/test/util.py b/Lib/importlib/test/util.py index 93b7cd2861b..ef32f7d690f 100644 --- a/Lib/importlib/test/util.py +++ b/Lib/importlib/test/util.py @@ -35,7 +35,7 @@ def uncache(*names): for name in names: if name in ('sys', 'marshal', 'imp'): raise ValueError( - "cannot uncache {0} as it will break _importlib".format(name)) + "cannot uncache {0}".format(name)) try: del sys.modules[name] except KeyError: @@ -124,7 +124,11 @@ class mock_modules: else: sys.modules[fullname] = self.modules[fullname] if fullname in self.module_code: - self.module_code[fullname]() + try: + self.module_code[fullname]() + except Exception: + del sys.modules[fullname] + raise return self.modules[fullname] def __enter__(self): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 7b44fa1344c..13164371025 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -3,3 +3,19 @@ from ._bootstrap import module_for_loader from ._bootstrap import set_loader from ._bootstrap import set_package +from ._bootstrap import _resolve_name + + +def resolve_name(name, package): + """Resolve a relative module name to an absolute one.""" + if not name.startswith('.'): + return name + elif not package: + raise ValueError('{!r} is not a relative name ' + '(no leading dot)'.format(name)) + level = 0 + for character in name: + if character != '.': + break + level += 1 + return _resolve_name(name[level:], package, level) |