Changed EvPopen to raise the child execption no matter what

cloudcontrol/node/utils.py

 import os
+import gc
 import sys
+import types
 import errno
 import signal
+import pickle
 import logging
 import resource
 import threading
+import traceback
 import subprocess
 from collections import deque
 from functools import wraps
+from subprocess import _eintr_retry_call
 
 import pyev
 
@@ -47,17 +52,9 @@ class EvPopen(subprocess.Popen):
         """
         self.main = main_loop
 
-        try:
-            subprocess.Popen.__init__(self, *args, **kwargs)
-        except OSError as exc:
-            if exc.errno == errno.ECHILD:
-                # in case on some child failure on startup, subprocess will call
-                # waitpid on child, but there is a great chance that, since
-                # libev catchs SIGCHLD, the syscall is performed after libev
-                # waitpid or retried after interrupted, causing it to fail with
-                # ECHILD errno, in that latter case, we ignore the error and
-                # return early
-                raise RemoteExecutionError('Early child death')
+        # this could raise but don't worry about zombies, they will be collected
+        # by libev
+        subprocess.Popen.__init__(self, *args, **kwargs)
 
         # check stdout, stderr fileno and create watchers if needed
         self.stdout_watcher = self.stderr_watcher = None
@@ -147,6 +144,133 @@ class EvPopen(subprocess.Popen):
 
         return tuple(map(u''.join, (self._stdout_output, self._stderr_output)))
 
+    # This is basically a copy-paste from stdlib subprocess module to
+    # prevent calling waitpid which would race with libev and would raise
+    # ECHILD
+    def _execute_child(self, args, executable, preexec_fn, close_fds,
+                       cwd, env, universal_newlines,
+                       startupinfo, creationflags, shell,
+                       p2cread, p2cwrite,
+                       c2pread, c2pwrite,
+                       errread, errwrite):
+        """Execute program (POSIX version)"""
+
+        if isinstance(args, types.StringTypes):
+            args = [args]
+        else:
+            args = list(args)
+
+        if shell:
+            args = ["/bin/sh", "-c"] + args
+
+        if executable is None:
+            executable = args[0]
+
+        # For transferring possible exec failure from child to parent
+        # The first char specifies the exception type: 0 means
+        # OSError, 1 means some other error.
+        errpipe_read, errpipe_write = os.pipe()
+        try:
+            try:
+                self._set_cloexec_flag(errpipe_write)
+
+                gc_was_enabled = gc.isenabled()
+                # Disable gc to avoid bug where gc -> file_dealloc ->
+                # write to stderr -> hang.  http://bugs.python.org/issue1336
+                gc.disable()
+                try:
+                    self.pid = os.fork()
+                except:
+                    if gc_was_enabled:
+                        gc.enable()
+                    raise
+                self._child_created = True
+                if self.pid == 0:
+                    # Child
+                    try:
+                        # Close parent's pipe ends
+                        if p2cwrite is not None:
+                            os.close(p2cwrite)
+                        if c2pread is not None:
+                            os.close(c2pread)
+                        if errread is not None:
+                            os.close(errread)
+                        os.close(errpipe_read)
+
+                        # Dup fds for child
+                        if p2cread is not None:
+                            os.dup2(p2cread, 0)
+                        if c2pwrite is not None:
+                            os.dup2(c2pwrite, 1)
+                        if errwrite is not None:
+                            os.dup2(errwrite, 2)
+
+                        # Close pipe fds.  Make sure we don't close the same
+                        # fd more than once, or standard fds.
+                        if p2cread is not None and p2cread not in (0,):
+                            os.close(p2cread)
+                        if c2pwrite is not None and c2pwrite not in (p2cread, 1):
+                            os.close(c2pwrite)
+                        if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
+                            os.close(errwrite)
+
+                        # Close all other fds, if asked for
+                        if close_fds:
+                            self._close_fds(but=errpipe_write)
+
+                        if cwd is not None:
+                            os.chdir(cwd)
+
+                        if preexec_fn:
+                            preexec_fn()
+
+                        if env is None:
+                            os.execvp(executable, args)
+                        else:
+                            os.execvpe(executable, args, env)
+
+                    except:
+                        exc_type, exc_value, tb = sys.exc_info()
+                        # Save the traceback and attach it to the exception object
+                        exc_lines = traceback.format_exception(exc_type,
+                                                               exc_value,
+                                                               tb)
+                        exc_value.child_traceback = ''.join(exc_lines)
+                        os.write(errpipe_write, pickle.dumps(exc_value))
+
+                    # This exitcode won't be reported to applications, so it
+                    # really doesn't matter what we return.
+                    os._exit(255)
+
+                # Parent
+                if gc_was_enabled:
+                    gc.enable()
+            finally:
+                # be sure the FD is closed no matter what
+                os.close(errpipe_write)
+
+            if p2cread is not None and p2cwrite is not None:
+                os.close(p2cread)
+            if c2pwrite is not None and c2pread is not None:
+                os.close(c2pwrite)
+            if errwrite is not None and errread is not None:
+                os.close(errwrite)
+
+            # Wait for exec to fail or succeed; possibly raising exception
+            # Exception limited to 1M
+            data = _eintr_retry_call(os.read, errpipe_read, 1048576)
+        finally:
+            # be sure the FD is closed no matter what
+            os.close(errpipe_read)
+
+        if data != "":
+            # _eintr_retry_call(os.waitpid, self.pid, 0)
+            child_exception = pickle.loads(data)
+            for fd in (p2cwrite, c2pread, errread):
+                if fd is not None:
+                    os.close(fd)
+            raise child_exception
+
     def wait(self):
         self.process_done.wait()
         return self.returncode