RPC deconnection error handling.

When the node is disconnected from the cc-server, it keeps it internal state
until the connection pass, then in case the same role is returned by the
cc-server, it just reregister its objects and tags, else it remove the previous
main plugin and recreate a new one corresponding to the adequate role.

ccnode/node.py

@@ -28,6 +28,9 @@ class RPCStartHandler(Thread):
     This class inherits from :class:`Thread` but only the connection part is run
     in a background thread.
 
+    Note:
+        As :class:`Thread`, it can be started only one time.
+
     """
     def __init__(self, loop):
         """
@@ -56,6 +59,7 @@ class RPCStartHandler(Thread):
                     port=self.loop.config.server_port,
                     handler=self.loop.rpc_handler,
                     loop=self.loop.evloop,
+                    on_disconnect=self.loop.restart_rpc_connection,
                 )
                 logger.debug('Connection created')
             except IOError:
@@ -121,27 +125,47 @@ class RPCStartHandler(Thread):
             return
 
         # set handler according to which role was returned by the cc-server
-        if response == u'host':
+        if response == self.loop.role and response is not None:
+            # we don't need to reload the plugins
+            # but we need to register the objects and tags
+            self.loop.tag_db.rpc_register()
+        elif response == u'host':
+            # close previous plugins if needed
+            if self.loop.role is not None:
+                self.loop.close_plugins()
+                # re-register the tags of the main loop
+                self.loop.tag_db.register()
             logger.debug('Role host affected')
             from ccnode.host import Handler as HostHandler
-            self.loop.role = HostHandler(loop=self.loop)
+            self.loop.main_plugin = HostHandler(loop=self.loop)
+            self.loop.role = u'host'
+            self.loop.register_plugin(self.loop.main_plugin)
         elif response == u'hv':
+            # close previous plugins if needed
+            if self.loop.role is not None:
+                self.loop.close_plugins()
+                # re-register the tags of the main loop
+                self.loop.tag_db.register()
             logger.debug('Role hypervisor affected')
+            # we don't import those modules at the top because some dependancies
+            # may not be installed
             from ccnode.hypervisor import Handler as HypervisorHandler
-            self.loop.role = HypervisorHandler(
+            self.loop.main_plugin = HypervisorHandler(
                 hypervisor_name=self.loop.config.server_user,
                 loop=self.loop,
             )
+            self.loop.role = u'hv'
+            self.loop.register_plugin(self.loop.main_plugin)
         else:
             logger.error('Failed authentication, role returned: %s', response)
             self.loop.role = None
             self.auth_id = None
-            return
+            # we also close the previously opened plugins
+            self.loop.close_plugins()
+            return  # we retry while it fails
 
-        if self.loop.role is not None:
-            self.stop()
-            logger.info('Successfully authenticated with role %s', response)
-            self.loop.register_plugin(self.loop.role)
+        self.stop()
+        logger.info('Successfully authenticated with role %s', response)
         self.auth_id = None
 
 
@@ -176,6 +200,7 @@ class MainLoop(object):
 
         # role
         self.role = None
+        self.main_plugin = None
 
         # tag database
         self.tag_db = TagDB(self, tags=DEFAULT_TAGS)
@@ -255,6 +280,19 @@ class MainLoop(object):
 
         plugin.stop()
 
+    def restart_rpc_connection(self, *args):
+        if not self.rpc_connected:
+            return
+
+        logger.error('Lost connection to the cc-server, will attempt'
+                     ' reconnection')
+
+        # clear connection
+        self.rpc_con = None
+        # attempt to connect to the cc-server again
+        self.connect = RPCStartHandler(self)
+        self.connect.start()
+
     def start(self):
         logger.info('Starting node')
         for signal in self.signals.itervalues():
@@ -270,13 +308,17 @@ class MainLoop(object):
             self.connect.stop()
         # close rpc
         if self.rpc_con is not None:
+            # disable callback to prevent trampoline calls
+            self.rpc_con._on_disconnect = None   # FIXME doesn't work
             self.rpc_con.shutdown()
         # close all plugins
         for plugin in self.registered_plugins:
             plugin.stop()
         self.registered_plugins = set()
 
+        # FIXME check for closing of main tags that were not in plugin
         self.role = None
+        self.main_plugin = None
         self.evloop.stop()
 
     def reload(self, watcher=None, revents=None):