Loading cloudcontrol/node/utils.py +135 −11 Original line number Diff line number Diff line import os import gc import sys import types import errno import signal import pickle import logging import resource import threading import traceback import subprocess from collections import deque from functools import wraps from subprocess import _eintr_retry_call import pyev Loading Loading @@ -47,17 +52,9 @@ class EvPopen(subprocess.Popen): """ self.main = main_loop try: # this could raise but don't worry about zombies, they will be collected # by libev subprocess.Popen.__init__(self, *args, **kwargs) except OSError as exc: if exc.errno == errno.ECHILD: # in case on some child failure on startup, subprocess will call # waitpid on child, but there is a great chance that, since # libev catchs SIGCHLD, the syscall is performed after libev # waitpid or retried after interrupted, causing it to fail with # ECHILD errno, in that latter case, we ignore the error and # return early raise RemoteExecutionError('Early child death') # check stdout, stderr fileno and create watchers if needed self.stdout_watcher = self.stderr_watcher = None Loading Loading @@ -147,6 +144,133 @@ class EvPopen(subprocess.Popen): return tuple(map(u''.join, (self._stdout_output, self._stderr_output))) # This is basically a copy-paste from stdlib subprocess module to # prevent calling waitpid which would race with libev and would raise # ECHILD def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (POSIX version)""" if isinstance(args, types.StringTypes): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable is None: executable = args[0] # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = os.pipe() try: try: self._set_cloexec_flag(errpipe_write) gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = os.fork() except: if gc_was_enabled: gc.enable() raise self._child_created = True if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # Dup fds for child if p2cread is not None: os.dup2(p2cread, 0) if c2pwrite is not None: os.dup2(c2pwrite, 1) if errwrite is not None: os.dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the same # fd more than once, or standard fds. if p2cread is not None and p2cread not in (0,): os.close(p2cread) if c2pwrite is not None and c2pwrite not in (p2cread, 1): os.close(c2pwrite) if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2): os.close(errwrite) # Close all other fds, if asked for if close_fds: self._close_fds(but=errpipe_write) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) # This exitcode won't be reported to applications, so it # really doesn't matter what we return. os._exit(255) # Parent if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) if p2cread is not None and p2cwrite is not None: os.close(p2cread) if c2pwrite is not None and c2pread is not None: os.close(c2pwrite) if errwrite is not None and errread is not None: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising exception # Exception limited to 1M data = _eintr_retry_call(os.read, errpipe_read, 1048576) finally: # be sure the FD is closed no matter what os.close(errpipe_read) if data != "": # _eintr_retry_call(os.waitpid, self.pid, 0) child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def wait(self): self.process_done.wait() return self.returncode Loading Loading
cloudcontrol/node/utils.py +135 −11 Original line number Diff line number Diff line import os import gc import sys import types import errno import signal import pickle import logging import resource import threading import traceback import subprocess from collections import deque from functools import wraps from subprocess import _eintr_retry_call import pyev Loading Loading @@ -47,17 +52,9 @@ class EvPopen(subprocess.Popen): """ self.main = main_loop try: # this could raise but don't worry about zombies, they will be collected # by libev subprocess.Popen.__init__(self, *args, **kwargs) except OSError as exc: if exc.errno == errno.ECHILD: # in case on some child failure on startup, subprocess will call # waitpid on child, but there is a great chance that, since # libev catchs SIGCHLD, the syscall is performed after libev # waitpid or retried after interrupted, causing it to fail with # ECHILD errno, in that latter case, we ignore the error and # return early raise RemoteExecutionError('Early child death') # check stdout, stderr fileno and create watchers if needed self.stdout_watcher = self.stderr_watcher = None Loading Loading @@ -147,6 +144,133 @@ class EvPopen(subprocess.Popen): return tuple(map(u''.join, (self._stdout_output, self._stderr_output))) # This is basically a copy-paste from stdlib subprocess module to # prevent calling waitpid which would race with libev and would raise # ECHILD def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (POSIX version)""" if isinstance(args, types.StringTypes): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable is None: executable = args[0] # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = os.pipe() try: try: self._set_cloexec_flag(errpipe_write) gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = os.fork() except: if gc_was_enabled: gc.enable() raise self._child_created = True if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # Dup fds for child if p2cread is not None: os.dup2(p2cread, 0) if c2pwrite is not None: os.dup2(c2pwrite, 1) if errwrite is not None: os.dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the same # fd more than once, or standard fds. if p2cread is not None and p2cread not in (0,): os.close(p2cread) if c2pwrite is not None and c2pwrite not in (p2cread, 1): os.close(c2pwrite) if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2): os.close(errwrite) # Close all other fds, if asked for if close_fds: self._close_fds(but=errpipe_write) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) # This exitcode won't be reported to applications, so it # really doesn't matter what we return. os._exit(255) # Parent if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) if p2cread is not None and p2cwrite is not None: os.close(p2cread) if c2pwrite is not None and c2pread is not None: os.close(c2pwrite) if errwrite is not None and errread is not None: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising exception # Exception limited to 1M data = _eintr_retry_call(os.read, errpipe_read, 1048576) finally: # be sure the FD is closed no matter what os.close(errpipe_read) if data != "": # _eintr_retry_call(os.waitpid, self.pid, 0) child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def wait(self): self.process_done.wait() return self.returncode Loading