diff options
Diffstat (limited to 'Lib/subprocess.py')
| -rw-r--r-- | Lib/subprocess.py | 851 |
1 files changed, 488 insertions, 363 deletions
diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 19a51889ea1..f32f081e97d 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -2,8 +2,6 @@ # # For more information about this module, see PEP 324. # -# This module should remain compatible with Python 2.2, see PEP 291. -# # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> # # Licensed to PSF under a Contributor Agreement. @@ -17,9 +15,6 @@ intends to replace several other, older modules and functions, like: os.system os.spawn* -os.popen* -popen2.* -commands.* Information about how the subprocess module can be used to replace these modules and functions can be found below. @@ -32,9 +27,10 @@ This module defines one class called Popen: class Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, + preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, pass_fds=()): Arguments are: @@ -43,12 +39,12 @@ args should be a string, or a sequence of program arguments. The program to execute is normally the first item in the args sequence or string, but can be explicitly set by using the executable argument. -On UNIX, with shell=False (default): In this case, the Popen class +On POSIX, with shell=False (default): In this case, the Popen class uses os.execvp() to execute the child program. args should normally be a sequence. A string will be treated as a sequence with the string as the only item (the program to execute). -On UNIX, with shell=True: If args is a string, it specifies the +On POSIX, with shell=True: If args is a string, it specifies the command string to execute through the shell. If args is a sequence, the first item specifies the command string, and any additional items will be treated as additional shell arguments. @@ -77,11 +73,19 @@ parent. Additionally, stderr can be STDOUT, which indicates that the stderr data from the applications should be captured into the same file handle as for stdout. -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. +On POSIX, if preexec_fn is set to a callable object, this object will be +called in the child process just before the child is executed. The use +of preexec_fn is not thread safe, using it in the presence of threads +could lead to a deadlock in the child process before the new executable +is executed. If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. +closed before the child process is executed. The default for close_fds +varies by platform: Always true on POSIX. True when stdin/stdout/stderr +are None on Windows, false otherwise. + +pass_fds is an optional sequence of file descriptors to keep open between the +parent and child. Providing any pass_fds implicitly sets close_fds to true. if shell is true, the specified command will be executed through the shell. @@ -89,12 +93,20 @@ shell. If cwd is not None, the current directory will be changed to cwd before the child is executed. +On POSIX, if restore_signals is True all signals that Python sets to +SIG_IGN are restored to SIG_DFL in the child process before the exec. +Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. This +parameter does nothing on Windows. + +On POSIX, if start_new_session is True, the setsid() system call will be made +in the child process prior to executing the command. + If env is not None, it defines the environment variables for the new process. If universal_newlines is true, the file objects stdout and stderr are opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or +the Unix end-of-line convention, '\r', the old Macintosh convention or '\r\n', the Windows convention. All of these external representations are seen as '\n' by the Python program. Note: This feature is only available if Python is built with universal newline support (the @@ -115,7 +127,7 @@ call(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - retcode = call(["ls", "-l"]) + >>> retcode = subprocess.call(["ls", "-l"]) check_call(*popenargs, **kwargs): Run command with arguments. Wait for command to complete. If the @@ -125,7 +137,33 @@ check_call(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - check_call(["ls", "-l"]) + >>> subprocess.check_call(["ls", "-l"]) + 0 + +getstatusoutput(cmd): + Return (status, output) of executing cmd in a shell. + + Execute the string 'cmd' in a shell with os.popen() and return a 2-tuple + (status, output). cmd is actually run as '{ cmd ; } 2>&1', so that the + returned output will contain output or error messages. A trailing newline + is stripped from the output. The exit status for the command can be + interpreted according to the rules for the C function wait(). Example: + + >>> subprocess.getstatusoutput('ls /bin/ls') + (0, '/bin/ls') + >>> subprocess.getstatusoutput('cat /bin/junk') + (256, 'cat: /bin/junk: No such file or directory') + >>> subprocess.getstatusoutput('/bin/junk') + (256, 'sh: /bin/junk: not found') + +getoutput(cmd): + Return output (stdout or stderr) of executing cmd in a shell. + + Like getstatusoutput(), except the exit status is ignored and the return + value is a string containing the command's output. Example: + + >>> subprocess.getoutput('ls /bin/ls') + '/bin/ls' check_output(*popenargs, **kwargs): Run command with arguments and return its output as a byte string. @@ -136,7 +174,7 @@ check_output(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: - output = check_output(["ls", "-l", "/dev/null"]) + >>> output = subprocess.check_output(["ls", "-l", "/dev/null"]) Exceptions @@ -209,7 +247,7 @@ pid returncode The child return code. A None value indicates that the process hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). + child was terminated by signal N (POSIX only). Replacing older functions with the subprocess module @@ -260,11 +298,11 @@ A more real-world example would look like this: try: retcode = call("mycmd" + " myarg", shell=True) if retcode < 0: - print >>sys.stderr, "Child was terminated by signal", -retcode + print("Child was terminated by signal", -retcode, file=sys.stderr) else: - print >>sys.stderr, "Child returned", retcode -except OSError, e: - print >>sys.stderr, "Execution failed:", e + print("Child returned", retcode, file=sys.stderr) +except OSError as e: + print("Execution failed:", e, file=sys.stderr) Replacing os.spawn* @@ -295,107 +333,18 @@ Environment example: os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) ==> Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - -Replacing os.popen* -------------------- -pipe = os.popen("cmd", mode='r', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdout=PIPE).stdout - -pipe = os.popen("cmd", mode='w', bufsize) -==> -pipe = Popen("cmd", shell=True, bufsize=bufsize, stdin=PIPE).stdin - - -(child_stdin, child_stdout) = os.popen2("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - - -(child_stdin, - child_stdout, - child_stderr) = os.popen3("cmd", mode, bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -(child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - - -(child_stdin, child_stdout_and_stderr) = os.popen4("cmd", mode, - bufsize) -==> -p = Popen("cmd", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) -(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - -On Unix, os.popen2, os.popen3 and os.popen4 also accept a sequence as -the command to execute, in which case arguments will be passed -directly to the program without shell intervention. This usage can be -replaced as follows: - -(child_stdin, child_stdout) = os.popen2(["/bin/ls", "-l"], mode, - bufsize) -==> -p = Popen(["/bin/ls", "-l"], bufsize=bufsize, stdin=PIPE, stdout=PIPE) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - -Return code handling translates as follows: - -pipe = os.popen("cmd", 'w') -... -rc = pipe.close() -if rc is not None and rc % 256: - print "There were some errors" -==> -process = Popen("cmd", 'w', shell=True, stdin=PIPE) -... -process.stdin.close() -if process.wait() != 0: - print "There were some errors" - - -Replacing popen2.* ------------------- -(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) -==> -p = Popen(["somestring"], shell=True, bufsize=bufsize - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -On Unix, popen2 also accepts a sequence as the command to execute, in -which case arguments will be passed directly to the program without -shell intervention. This usage can be replaced as follows: - -(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, - mode) -==> -p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen, -except that: - -* subprocess.Popen raises an exception if the execution fails -* the capturestderr argument is replaced with the stderr argument. -* stdin=PIPE and stdout=PIPE must be specified. -* popen2 closes all filedescriptors by default, but you have to specify - close_fds=True with subprocess.Popen. """ import sys mswindows = (sys.platform == "win32") +import io import os -import types import traceback import gc import signal +import builtins +import warnings import errno # Exception classes used by this module. @@ -431,14 +380,39 @@ else: import fcntl import pickle + try: + import _posixsubprocess + except ImportError: + _posixsubprocess = None + warnings.warn("The _posixsubprocess module is not being used. " + "Child process reliability may suffer if your " + "program uses threads.", RuntimeWarning) + # When select or poll has indicated that the file is writable, # we can write up to _PIPE_BUF bytes without risk of blocking. # POSIX defines PIPE_BUF as >= 512. _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) + _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1) -__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", - "check_output", "CalledProcessError"] + def _set_cloexec(fd, cloexec): + old = fcntl.fcntl(fd, fcntl.F_GETFD) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC) + + if _posixsubprocess: + _create_pipe = _posixsubprocess.cloexec_pipe + else: + def _create_pipe(): + fds = os.pipe() + _set_cloexec(fds[0], True) + _set_cloexec(fds[1], True) + return fds + +__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", + "getoutput", "check_output", "CalledProcessError"] if mswindows: from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, @@ -455,11 +429,15 @@ try: except: MAXFD = 256 +# This lists holds Popen instances for which the underlying process had not +# exited at the time its __del__ method got called: those processes are wait()ed +# for synchronously from _cleanup() when a new Popen object is created, to avoid +# zombie processes. _active = [] def _cleanup(): for inst in _active[:]: - res = inst._internal_poll(_deadstate=sys.maxint) + res = inst._internal_poll(_deadstate=sys.maxsize) if res is not None: try: _active.remove(inst) @@ -522,7 +500,7 @@ def check_output(*popenargs, **kwargs): The arguments are the same as for the Popen constructor. Example: >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' The stdout argument is not allowed as it is used internally. To capture standard error in the result, use stderr=STDOUT. @@ -530,11 +508,11 @@ def check_output(*popenargs, **kwargs): >>> check_output(["/bin/sh", "-c", ... "ls -l non_existent_file ; exit 0"], ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' + b'ls: non_existent_file: No such file or directory\n' """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') - process = Popen(stdout=PIPE, *popenargs, **kwargs) + process = Popen(*popenargs, stdout=PIPE, **kwargs) output, unused_err = process.communicate() retcode = process.poll() if retcode: @@ -615,29 +593,90 @@ def list2cmdline(seq): return ''.join(result) +# Various tools for executing commands and looking at their output and status. +# +# NB This only works (and is only relevant) for POSIX. + +def getstatusoutput(cmd): + """Return (status, output) of executing cmd in a shell. + + Execute the string 'cmd' in a shell with os.popen() and return a 2-tuple + (status, output). cmd is actually run as '{ cmd ; } 2>&1', so that the + returned output will contain output or error messages. A trailing newline + is stripped from the output. The exit status for the command can be + interpreted according to the rules for the C function wait(). Example: + + >>> import subprocess + >>> subprocess.getstatusoutput('ls /bin/ls') + (0, '/bin/ls') + >>> subprocess.getstatusoutput('cat /bin/junk') + (256, 'cat: /bin/junk: No such file or directory') + >>> subprocess.getstatusoutput('/bin/junk') + (256, 'sh: /bin/junk: not found') + """ + pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r') + text = pipe.read() + sts = pipe.close() + if sts is None: sts = 0 + if text[-1:] == '\n': text = text[:-1] + return sts, text + + +def getoutput(cmd): + """Return output (stdout or stderr) of executing cmd in a shell. + + Like getstatusoutput(), except the exit status is ignored and the return + value is a string containing the command's output. Example: + + >>> import subprocess + >>> subprocess.getoutput('ls /bin/ls') + '/bin/ls' + """ + return getstatusoutput(cmd)[1] + + +_PLATFORM_DEFAULT_CLOSE_FDS = object() + + class Popen(object): def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): + preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, + shell=False, cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0, + restore_signals=True, start_new_session=False, + pass_fds=()): """Create new Popen instance.""" _cleanup() self._child_created = False - if not isinstance(bufsize, (int, long)): + if bufsize is None: + bufsize = 0 # Restore default + if not isinstance(bufsize, int): raise TypeError("bufsize must be an integer") if mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") - if close_fds and (stdin is not None or stdout is not None or - stderr is not None): - raise ValueError("close_fds is not supported on Windows " - "platforms if you redirect stdin/stdout/stderr") + any_stdio_set = (stdin is not None or stdout is not None or + stderr is not None) + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + if any_stdio_set: + close_fds = False + else: + close_fds = True + elif close_fds and any_stdio_set: + raise ValueError( + "close_fds is not supported on Windows platforms" + " if you redirect stdin/stdout/stderr") else: # POSIX + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + close_fds = True + if pass_fds and not close_fds: + warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) + close_fds = True if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") @@ -664,70 +703,89 @@ class Popen(object): # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None + # are -1 when not using PIPEs. The child objects are -1 # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) + # We wrap OS handles *before* launching the child, otherwise a + # quickly terminating child could make our fds unwrappable + # (see #8458). + + if mswindows: + if p2cwrite != -1: + p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) + if c2pread != -1: + c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) + if errread != -1: + errread = msvcrt.open_osfhandle(errread.Detach(), 0) + + if p2cwrite != -1: + self.stdin = io.open(p2cwrite, 'wb', bufsize) + if self.universal_newlines: + self.stdin = io.TextIOWrapper(self.stdin, write_through=True) + if c2pread != -1: + self.stdout = io.open(c2pread, 'rb', bufsize) + if universal_newlines: + self.stdout = io.TextIOWrapper(self.stdout) + if errread != -1: + self.stderr = io.open(errread, 'rb', bufsize) + if universal_newlines: + self.stderr = io.TextIOWrapper(self.stderr) + try: self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, + pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) - except Exception: - # Preserve original exception in case os.close raises. - exc_type, exc_value, exc_trace = sys.exc_info() + errread, errwrite, + restore_signals, start_new_session) + except: + # Cleanup if the child failed starting. + for f in filter(None, (self.stdin, self.stdout, self.stderr)): + try: + f.close() + except EnvironmentError: + pass # Ignore EBADF or other errors. + # Make sure the child pipes are closed as well. to_close = [] - # Only close the pipes we created. if stdin == PIPE: - to_close.extend((p2cread, p2cwrite)) + to_close.append(p2cread) if stdout == PIPE: - to_close.extend((c2pread, c2pwrite)) + to_close.append(c2pwrite) if stderr == PIPE: - to_close.extend((errread, errwrite)) - + to_close.append(errwrite) for fd in to_close: try: os.close(fd) except EnvironmentError: pass - raise exc_type, exc_value, exc_trace + raise - if mswindows: - if p2cwrite is not None: - p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) - if c2pread is not None: - c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) - if errread is not None: - errread = msvcrt.open_osfhandle(errread.Detach(), 0) - - if p2cwrite is not None: - self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) - if c2pread is not None: - if universal_newlines: - self.stdout = os.fdopen(c2pread, 'rU', bufsize) - else: - self.stdout = os.fdopen(c2pread, 'rb', bufsize) - if errread is not None: - if universal_newlines: - self.stderr = os.fdopen(errread, 'rU', bufsize) - else: - self.stderr = os.fdopen(errread, 'rb', bufsize) + def _translate_newlines(self, data, encoding): + data = data.decode(encoding) + return data.replace("\r\n", "\n").replace("\r", "\n") - def _translate_newlines(self, data): - data = data.replace("\r\n", "\n") - data = data.replace("\r", "\n") - return data + def __enter__(self): + return self + def __exit__(self, type, value, traceback): + if self.stdout: + self.stdout.close() + if self.stderr: + self.stderr.close() + if self.stdin: + self.stdin.close() + # Wait for the process to terminate, to avoid zombies. + self.wait() - def __del__(self, _maxint=sys.maxint, _active=_active): + def __del__(self, _maxsize=sys.maxsize, _active=_active): # If __init__ hasn't had a chance to execute (e.g. if it # was passed an undeclared keyword argument), we don't # have a _child_created attribute at all. @@ -735,7 +793,7 @@ class Popen(object): # We didn't get to successfully create a child process. return # In case the child hasn't been waited on, check if it's done. - self._internal_poll(_deadstate=_maxint) + self._internal_poll(_deadstate=_maxsize) if self.returncode is None and _active is not None: # Child is still running, keep us alive until we can wait on it. _active.append(self) @@ -788,11 +846,11 @@ class Popen(object): p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) + return (-1, -1, -1, -1, -1, -1) - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) @@ -865,20 +923,23 @@ class Popen(object): def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, + pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" - if not isinstance(args, types.StringTypes): + assert not pass_fds, "pass_fds not supported on Windows." + + if not isinstance(args, str): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() - if None not in (p2cread, c2pwrite, errwrite): + if -1 not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite @@ -901,7 +962,7 @@ class Popen(object): # cause random failures on win9x. Specifically a # dialog: "Your program accessed mem currently in # use at xxx" and a hopeful warning about the - # stability of your system. Cost is Ctrl+C wont + # stability of your system. Cost is Ctrl+C won't # kill children. creationflags |= _subprocess.CREATE_NEW_CONSOLE @@ -915,7 +976,7 @@ class Popen(object): env, cwd, startupinfo) - except pywintypes.error, e: + except pywintypes.error as e: # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really # translate errno using _sys_errlist (or similar), but @@ -928,11 +989,11 @@ class Popen(object): # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. - if p2cread is not None: + if p2cread != -1: p2cread.Close() - if c2pwrite is not None: + if c2pwrite != -1: c2pwrite.Close() - if errwrite is not None: + if errwrite != -1: errwrite.Close() # Retain the process handle, but close the thread handle @@ -970,6 +1031,7 @@ class Popen(object): def _readerthread(self, fh, buffer): buffer.append(fh.read()) + fh.close() def _communicate(self, input): @@ -980,13 +1042,13 @@ class Popen(object): stdout = [] stdout_thread = threading.Thread(target=self._readerthread, args=(self.stdout, stdout)) - stdout_thread.setDaemon(True) + stdout_thread.daemon = True stdout_thread.start() if self.stderr: stderr = [] stderr_thread = threading.Thread(target=self._readerthread, args=(self.stderr, stderr)) - stderr_thread.setDaemon(True) + stderr_thread.daemon = True stderr_thread.start() if self.stdin: @@ -1009,16 +1071,6 @@ class Popen(object): if stderr is not None: stderr = stderr[0] - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - self.wait() return (stdout, stderr) @@ -1059,14 +1111,14 @@ class Popen(object): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 if stdin is None: pass elif stdin == PIPE: - p2cread, p2cwrite = self.pipe_cloexec() + p2cread, p2cwrite = _create_pipe() elif isinstance(stdin, int): p2cread = stdin else: @@ -1076,7 +1128,7 @@ class Popen(object): if stdout is None: pass elif stdout == PIPE: - c2pread, c2pwrite = self.pipe_cloexec() + c2pread, c2pwrite = _create_pipe() elif isinstance(stdout, int): c2pwrite = stdout else: @@ -1086,7 +1138,7 @@ class Popen(object): if stderr is None: pass elif stderr == PIPE: - errread, errwrite = self.pipe_cloexec() + errread, errwrite = _create_pipe() elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): @@ -1100,54 +1152,26 @@ class Popen(object): errread, errwrite) - def _set_cloexec_flag(self, fd, cloexec=True): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - - def pipe_cloexec(self): - """Create a pipe with FDs set CLOEXEC.""" - # Pipes' FDs are set CLOEXEC by default because we don't want them - # to be inherited by other subprocesses: the CLOEXEC flag is removed - # from the child's FDs by _dup2(), between fork() and exec(). - # This is not atomic: we would need the pipe2() syscall for that. - r, w = os.pipe() - self._set_cloexec_flag(r) - self._set_cloexec_flag(w) - return r, w - - - def _close_fds(self, but): - if hasattr(os, 'closerange'): - os.closerange(3, but) - os.closerange(but + 1, MAXFD) - else: - for i in xrange(3, MAXFD): - if i == but: - continue - try: - os.close(i) - except: - pass + def _close_fds(self, fds_to_keep): + start_fd = 3 + for fd in sorted(fds_to_keep): + if fd >= start_fd: + os.closerange(start_fd, fd) + start_fd = fd + 1 + if start_fd <= MAXFD: + os.closerange(start_fd, MAXFD) def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, + pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite): + errread, errwrite, + restore_signals, start_new_session): """Execute program (POSIX version)""" - if isinstance(args, types.StringTypes): + if isinstance(args, str): args = [args] else: args = list(args) @@ -1159,122 +1183,217 @@ class Popen(object): if executable is None: executable = args[0] + orig_executable = executable - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = self.pipe_cloexec() + # For transferring possible exec failure from child to parent. + # Data format: "exception name:hex errno:description" + # Pickle is not used; it is complex and involves memory allocation. + errpipe_read, errpipe_write = _create_pipe() try: try: - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. http://bugs.python.org/issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite is not None: - os.close(p2cwrite) - if c2pread is not None: - os.close(c2pread) - if errread is not None: - os.close(errread) - os.close(errpipe_read) - - # When duping fds, if there arises a situation - # where one of the fds is either 0, 1 or 2, it - # is possible that it is overwritten (#12607). - if c2pwrite == 0: - c2pwrite = os.dup(c2pwrite) - if errwrite == 0 or errwrite == 1: - errwrite = os.dup(errwrite) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - self._set_cloexec_flag(a, False) - elif a is not None: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = { None } - for fd in [p2cread, c2pwrite, errwrite]: - if fd not in closed and fd > 2: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd is not None: - os.chdir(cwd) - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) + if _posixsubprocess: + # We must avoid complex work that could involve + # malloc or free in the child process to avoid + # potential deadlocks, thus we do all this here. + # and pass it to fork_exec() + + if env is not None: + env_list = [os.fsencode(k) + b'=' + os.fsencode(v) + for k, v in env.items()] + else: + env_list = None # Use execv instead of execve. + executable = os.fsencode(executable) + if os.path.dirname(executable): + executable_list = (executable,) + else: + # This matches the behavior of os._execvpe(). + executable_list = tuple( + os.path.join(os.fsencode(dir), executable) + for dir in os.get_exec_path(env)) + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self.pid = _posixsubprocess.fork_exec( + args, executable_list, + close_fds, sorted(fds_to_keep), cwd, env_list, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, preexec_fn) + self._child_created = True + else: + # Pure Python implementation: It is not thread safe. + # This implementation may deadlock in the child if your + # parent process has any other threads running. + + gc_was_enabled = gc.isenabled() + # Disable gc to avoid bug where gc -> file_dealloc -> + # write to stderr -> hang. See issue1336 + gc.disable() + try: + self.pid = os.fork() except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, so it - # really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() + if gc_was_enabled: + gc.enable() + raise + self._child_created = True + if self.pid == 0: + # Child + reached_preexec = False + try: + # Close parent's pipe ends + if p2cwrite != -1: + os.close(p2cwrite) + if c2pread != -1: + os.close(c2pread) + if errread != -1: + os.close(errread) + os.close(errpipe_read) + + # When duping fds, if there arises a situation + # where one of the fds is either 0, 1 or 2, it + # is possible that it is overwritten (#12607). + if c2pwrite == 0: + c2pwrite = os.dup(c2pwrite) + if errwrite == 0 or errwrite == 1: + errwrite = os.dup(errwrite) + + # Dup fds for child + def _dup2(a, b): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if a == b: + _set_cloexec(a, False) + elif a != -1: + os.dup2(a, b) + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) + + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = set() + for fd in [p2cread, c2pwrite, errwrite]: + if fd > 2 and fd not in closed: + os.close(fd) + closed.add(fd) + + # Close all other fds, if asked for + if close_fds: + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self._close_fds(fds_to_keep) + + + if cwd is not None: + os.chdir(cwd) + + # This is a copy of Python/pythonrun.c + # _Py_RestoreSignals(). If that were exposed + # as a sys._py_restoresignals func it would be + # better.. but this pure python implementation + # isn't likely to be used much anymore. + if restore_signals: + signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') + for sig in signals: + if hasattr(signal, sig): + signal.signal(getattr(signal, sig), + signal.SIG_DFL) + + if start_new_session and hasattr(os, 'setsid'): + os.setsid() + + reached_preexec = True + if preexec_fn: + preexec_fn() + + if env is None: + os.execvp(executable, args) + else: + os.execvpe(executable, args, env) + + except: + try: + exc_type, exc_value = sys.exc_info()[:2] + if isinstance(exc_value, OSError): + errno_num = exc_value.errno + else: + errno_num = 0 + if not reached_preexec: + exc_value = "noexec" + message = '%s:%x:%s' % (exc_type.__name__, + errno_num, exc_value) + message = message.encode(errors="surrogatepass") + os.write(errpipe_write, message) + except Exception: + # We MUST not allow anything odd happening + # above to prevent us from exiting below. + pass + + # This exitcode won't be reported to applications + # so it really doesn't matter what we return. + os._exit(255) + + # Parent + if gc_was_enabled: + gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) - if p2cread is not None and p2cwrite is not None: + if p2cread != -1 and p2cwrite != -1: os.close(p2cread) - if c2pwrite is not None and c2pread is not None: + if c2pwrite != -1 and c2pread != -1: os.close(c2pwrite) - if errwrite is not None and errread is not None: + if errwrite != -1 and errread != -1: os.close(errwrite) - # Wait for exec to fail or succeed; possibly raising exception - # Exception limited to 1M - data = _eintr_retry_call(os.read, errpipe_read, 1048576) + # Wait for exec to fail or succeed; possibly raising an + # exception (limited in size) + errpipe_data = bytearray() + while True: + part = _eintr_retry_call(os.read, errpipe_read, 50000) + errpipe_data += part + if not part or len(errpipe_data) > 50000: + break finally: # be sure the FD is closed no matter what os.close(errpipe_read) - if data != "": + if errpipe_data: try: _eintr_retry_call(os.waitpid, self.pid, 0) except OSError as e: if e.errno != errno.ECHILD: raise - child_exception = pickle.loads(data) - raise child_exception + try: + exception_name, hex_errno, err_msg = ( + errpipe_data.split(b':', 2)) + except ValueError: + exception_name = b'RuntimeError' + hex_errno = b'0' + err_msg = (b'Bad exception data from child: ' + + repr(errpipe_data)) + child_exception_type = getattr( + builtins, exception_name.decode('ascii'), + RuntimeError) + err_msg = err_msg.decode(errors="surrogatepass") + if issubclass(child_exception_type, OSError) and hex_errno: + errno_num = int(hex_errno, 16) + child_exec_never_called = (err_msg == "noexec") + if child_exec_never_called: + err_msg = "" + if errno_num != 0: + err_msg = os.strerror(errno_num) + if errno_num == errno.ENOENT: + if child_exec_never_called: + # The error must be from chdir(cwd). + err_msg += ': ' + repr(cwd) + else: + err_msg += ': ' + repr(orig_executable) + raise child_exception_type(errno_num, err_msg) + raise child_exception_type(err_msg) def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, @@ -1308,7 +1427,7 @@ class Popen(object): except _os_error as e: if _deadstate is not None: self.returncode = _deadstate - if e.errno == _ECHILD: + elif e.errno == _ECHILD: # This happens if SIGCLD is set to be ignored or # waiting for child processes has otherwise been # disabled for our process. This child is dead, we @@ -1354,19 +1473,19 @@ class Popen(object): # All data exchanged. Translate lists into strings. if stdout is not None: - stdout = ''.join(stdout) + stdout = b''.join(stdout) if stderr is not None: - stderr = ''.join(stderr) - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(file, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) + stderr = b''.join(stderr) + + # Translate newlines, if requested. + # This also turns bytes into strings. + if self.universal_newlines: + if stdout is not None: + stdout = self._translate_newlines(stdout, + self.stdout.encoding) + if stderr is not None: + stderr = self._translate_newlines(stderr, + self.stderr.encoding) self.wait() return (stdout, stderr) @@ -1403,11 +1522,14 @@ class Popen(object): while fd2file: try: ready = poller.poll() - except select.error, e: + except select.error as e: if e.args[0] == errno.EINTR: continue raise + # XXX Rewrite these to use non-blocking I/O on the + # file objects; they are no longer using C stdio! + for fd, mode in ready: if mode & select.POLLOUT: chunk = input[input_offset : input_offset + _PIPE_BUF] @@ -1452,11 +1574,14 @@ class Popen(object): while read_set or write_set: try: rlist, wlist, xlist = select.select(read_set, write_set, []) - except select.error, e: + except select.error as e: if e.args[0] == errno.EINTR: continue raise + # XXX Rewrite these to use non-blocking I/O on the + # file objects; they are no longer using C stdio! + if self.stdin in wlist: chunk = input[input_offset : input_offset + _PIPE_BUF] try: @@ -1475,14 +1600,14 @@ class Popen(object): if self.stdout in rlist: data = os.read(self.stdout.fileno(), 1024) - if data == "": + if not data: self.stdout.close() read_set.remove(self.stdout) stdout.append(data) if self.stderr in rlist: data = os.read(self.stderr.fileno(), 1024) - if data == "": + if not data: self.stderr.close() read_set.remove(self.stderr) stderr.append(data) @@ -1511,8 +1636,8 @@ def _demo_posix(): # Example 1: Simple redirection: Get process list # plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print "Process list:" - print plist + print("Process list:") + print(plist) # # Example 2: Change uid before executing child @@ -1524,42 +1649,42 @@ def _demo_posix(): # # Example 3: Connecting several subprocesses # - print "Looking for 'hda'..." + print("Looking for 'hda'...") p1 = Popen(["dmesg"], stdout=PIPE) p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) + print(repr(p2.communicate()[0])) # # Example 4: Catch execution error # - print - print "Trying a weird file..." + print() + print("Trying a weird file...") try: - print Popen(["/this/path/does/not/exist"]).communicate() - except OSError, e: + print(Popen(["/this/path/does/not/exist"]).communicate()) + except OSError as e: if e.errno == errno.ENOENT: - print "The file didn't exist. I thought so..." - print "Child traceback:" - print e.child_traceback + print("The file didn't exist. I thought so...") + print("Child traceback:") + print(e.child_traceback) else: - print "Error", e.errno + print("Error", e.errno) else: - print >>sys.stderr, "Gosh. No error." + print("Gosh. No error.", file=sys.stderr) def _demo_windows(): # # Example 1: Connecting several subprocesses # - print "Looking for 'PROMPT' in set output..." + print("Looking for 'PROMPT' in set output...") p1 = Popen("set", stdout=PIPE, shell=True) p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) + print(repr(p2.communicate()[0])) # # Example 2: Simple execution of program # - print "Executing calc..." + print("Executing calc...") p = Popen("calc") p.wait() |