From 7e731b868b7c2514ef09bd8c2e698274ca91ebf7 Mon Sep 17 00:00:00 2001 From: Thibault VINCENT Date: Tue, 6 Sep 2011 17:37:51 +0200 Subject: [PATCH] initial import of new codebase --- bin/cc-node | 230 +------ ccnode/__init__.py | 51 +- ccnode/ccnode.py | 102 ---- ccnode/common.py | 359 ----------- ccnode/config.py | 193 ++++++ ccnode/drbd.py | 329 ---------- ccnode/errors.py | 118 ---- ccnode/handlers.py | 979 ------------------------------ ccnode/jobs.py | 1157 +----------------------------------- ccnode/kernel.py | 655 ++++++++++++++++++++ ccnode/kvm.py | 28 - ccnode/launcher.py | 106 ++++ ccnode/libvirtwrapper.py | 831 -------------------------- ccnode/logging.py | 159 +++++ ccnode/lvm.py | 343 ----------- ccnode/modules/__init__.py | 29 + ccnode/modules/execute.py | 72 +++ ccnode/modules/shutdown.py | 73 +++ ccnode/threading.py | 100 ++++ ccnode/utils.py | 210 ------- ccnode/xen.py | 50 -- debian/init | 142 ----- doc/Makefile | 89 --- doc/api.rst | 52 -- doc/conf.py | 205 ------- doc/index.rst | 23 - doc/libvirt.rst | 43 -- etc/cc-node.conf | 28 +- 28 files changed, 1494 insertions(+), 5262 deletions(-) delete mode 100644 ccnode/ccnode.py delete mode 100644 ccnode/common.py create mode 100644 ccnode/config.py delete mode 100644 ccnode/drbd.py delete mode 100644 ccnode/errors.py delete mode 100644 ccnode/handlers.py create mode 100644 ccnode/kernel.py delete mode 100644 ccnode/kvm.py create mode 100644 ccnode/launcher.py delete mode 100644 ccnode/libvirtwrapper.py create mode 100644 ccnode/logging.py delete mode 100644 ccnode/lvm.py create mode 100644 ccnode/modules/__init__.py create mode 100644 ccnode/modules/execute.py create mode 100644 ccnode/modules/shutdown.py create mode 100644 ccnode/threading.py delete mode 100644 ccnode/utils.py delete mode 100644 ccnode/xen.py delete mode 100644 debian/init delete mode 100644 doc/Makefile delete mode 100644 doc/api.rst delete mode 100644 doc/conf.py delete mode 100644 doc/index.rst delete mode 100644 doc/libvirt.rst mode change 100644 => 100755 etc/cc-node.conf diff --git a/bin/cc-node b/bin/cc-node index 0c1e321..c3a9b0e 100755 --- a/bin/cc-node +++ b/bin/cc-node @@ -1,210 +1,28 @@ #!/usr/bin/env python - -import logging -import threading -import logging.handlers -import sys -import os -import atexit -import ConfigParser -from math import exp -from time import sleep -from optparse import OptionParser - -from daemon import DaemonContext -from sjrpc.core import RpcError - -from ccnode.ccnode import CCNode -from ccnode import __version__ - -MAX_AUTH_TIMEOUT = 30 -DEFAULT_CONFIG_FILE = '/etc/cc-node.conf' -DEFAULT_PID_FILE = '/var/run/cc-node.pid' -DEFAULT_CONFIGURATION = { - 'address': '', - 'login': '', - 'password': '', - 'port': '1984', - 'verbosity': '0', - 'detect_hypervisor': 'yes', - #'ssl_cert': '', - 'command_execution' : 'yes', - 'force_xen' : 'no', -} - -def authentication(node, suicide, login, password): - ''' - Node authentication thread - ''' - timeout = 1 - while not suicide.is_set() and node.get_rpc(): - logging.debug('Sending authentication request') - if node.authentify(login, password): - logging.error('Authentication suscessfull') - return - else: - logging.error('Authentication failure') - timeout += 0.1 - if timeout >= MAX_AUTH_TIMEOUT: - timeout = MAX_AUTH_TIMEOUT - sleep(exp(timeout)) - -def setup_logging(options, instance_id=None): - ''' - ''' - level = logging.ERROR - verb = int(options['verbosity']) - if verb: - if verb == 1: - level = logging.WARNING - elif verb == 2: - level = logging.INFO - else: - level = logging.DEBUG - logger = logging.getLogger() - logger.setLevel(level) - if options['stdout']: - handler = logging.StreamHandler() - else: - facility = logging.handlers.SysLogHandler.LOG_DAEMON - handler = logging.handlers.SysLogHandler(address='/dev/log', - facility=facility) - fmt = 'cc-node: %(levelname)s ' - if options['verbosity'] == 0: - fmt += '%(message)s' - else: - if isinstance(instance_id, int): - fmt += 'id=%i ' % instance_id - if options['stdout']: - fmt += ("\x1B[30;47m%(process)d\x1B[0m:" - "\x1B[30;42m%(threadName)s\x1B[0m:" - "\x1B[30;43m%(funcName)s\x1B[0m:" - "\x1B[30;44m%(lineno)d\x1B[0m@" - "\x1B[30;45m%(msecs)d" - "\x1B[0m\t\t " - "%(message)s") - else: - fmt += ("%(process)d:" - "%(threadName)s:" - "%(funcName)s:" - "%(lineno)d@" - "%(msecs)d " - "%(message)s") - handler.setFormatter(logging.Formatter(fmt)) - logger.handlers = [] - logger.addHandler(handler) - -def run_node(options): - ''' - ''' - # instance death event - suicide_event = threading.Event() - - try: - # setup logging infrastructure - setup_logging(options) - - # create client - logging.debug('Initializing client') - try: - node = CCNode(options['address'], int(options['port']), - options['detect_hypervisor'] == 'yes', - options['command_execution'] == 'yes', - force_xen=(options['force_xen'] == 'yes')) - except Exception as err: - logging.error('Client initialization failure: `%s`:`%s`', - repr(err), err) - raise err - - # reconfigure logging with instance ID - setup_logging(options, instance_id=id(node)) - - # auth thread - logging.debug('Starting authentication thread') - auth_thread = threading.Thread(target=authentication, name='Auth', - args=(node, suicide_event, options['login'], options['password'])) - auth_thread.daemon = True - auth_thread.start() - - # main loop - logging.debug('Starting main loop') - node.run() - except Exception as err: - logging.error('run_node: `%s` -> `%s`', repr(err), err) - try: - node.get_rpc().shutdown() - except: - pass - finally: - # ensure everything is killed properly - suicide_event.set() +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from ccnode.launcher import Launcher if __name__ == '__main__': - # configure argument parser - op = OptionParser(version='%%prog %s' % __version__) - op.add_option('-c', '--config', default=DEFAULT_CONFIG_FILE, - help='configuration file (default: %default)') - op.add_option('-d', '--daemonize', default=False, action='store_true', - help='run as daemon and write pid file') - op.add_option('-p', '--pidfile', default=DEFAULT_PID_FILE, - help='pid file (default: %default)') - op.add_option('-s', '--stdout', action='store_true', default=False, - help='log on standard output instead of syslog') - cliopts, args = op.parse_args() - - # read the config file - config = ConfigParser.SafeConfigParser() - config.read(cliopts.config) - try: - options = dict(config.items('node')) - except ConfigParser.NoSectionError: - sys.stderr.write('Configuration error: `node` section must exist ' - 'in `%s`\n' % cliopts.config) - sys.exit(1) - - # applying default config file options - for opt, default in DEFAULT_CONFIGURATION.iteritems(): - if opt not in options or not options[opt]: - if default is None: - sys.stderr.write('Configuration error: you must specify `%s` ' - 'option in `%s` !\n' % (opt, cliopts.config)) - sys.exit(1) - else: - options[opt] = default - - # Merge command options and .conf file options - for opt in ('daemonize', 'pidfile', 'stdout'): - if getattr(cliopts, opt) is not None: - options[opt] = getattr(cliopts, opt) - - # Create option set for the daemonization process - daemon_opts = {} - daemonize = options['daemonize'] - if isinstance(daemonize, str): - daemonize = daemonize in ('yes', 'true') - daemon_opts['detach_process'] = daemonize - if not daemonize: - daemon_opts['stderr'] = sys.stderr - daemon_opts['stdout'] = sys.stderr - if options['pidfile']: - #daemon_opts['pidfile'] = lockfile.FileLock(options['pidfile']) - pidfile = open(cliopts.pidfile, 'w') - daemon_opts['files_preserve'] = [pidfile] - - with DaemonContext(**daemon_opts): - # write pid in pidfile - if pidfile is not None: - pidfile.write('%s' % os.getpid()) - pidfile.flush() - # register pidfile cleaning - @atexit.register - def clean_pidfile(): - pidfile.seek(0) - pidfile.truncate() - pidfile.flush() - # run node - while True: - run_node(options) - logging.critical('Critical error, restarting node !') - sleep(2) + Launcher().main() diff --git a/ccnode/__init__.py b/ccnode/__init__.py index 1444afb..769579a 100644 --- a/ccnode/__init__.py +++ b/ccnode/__init__.py @@ -1,10 +1,51 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# -""" +__product__ = 'Cloud-Control Node' +__version__ = '0~dev' +__canonical__ = 'cc-node' +__develmode__ = False -TODO : rewrite based on older doc -""" +def git_version(): + global __version__ + import os + import sys + from subprocess import Popen, PIPE, CalledProcessError + cwd = os.getcwd() + try: + os.chdir(os.path.dirname(sys.argv[0])) + p = Popen(["git", "log", "--pretty=format:%H" ], stdout=PIPE, + stderr=open("/dev/null", "wb")) + p.wait() + if p.returncode == 0: + githash = p.stdout.readline().strip() + if len(githash) > 0: + __version__ += "-git-%s" % githash + except OSError: + pass + finally: + os.chdir(cwd) -__version__ = '17' +if __version__.find("dev") != -1: + __develmode__ = True + git_version() diff --git a/ccnode/ccnode.py b/ccnode/ccnode.py deleted file mode 100644 index 3fd7945..0000000 --- a/ccnode/ccnode.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- - -import logging -from threading import Timer, Lock, Event -from time import sleep - -from sjrpc.utils import ConnectionProxy -from sjrpc.core import RpcError, RpcConnection -# FIXME relative import -import handlers - - -class CCNode(object): - ''' - Handle node initialization, connection to server, and authentication - ''' - def __init__(self, server, port, hypervisor, exec_cmd, force_xen=False, - cert=None): - ''' - ''' - self._scheduler_timer = None - self._scheduler_mutex = Lock() - self._scheduler_stopped = Event() - self._is_hv = hypervisor # hugly - self._is_xen = force_xen # hugly - self._exec_cmd = exec_cmd # hugly - self._handler = None - self._rpc = RpcConnection.from_addr_ssl(server, port, - handler=self._handler) - self._server = ConnectionProxy(self._rpc) - - def run(self): - ''' - ''' - try: - self._rpc.run() - except: - raise - finally: - self._scheduler_stopped.set() - - def authentify(self, login, password): - ''' - ''' - logging.debug('Authenticating user `%s` on connection `%i`' % (login, - id(self._server))) - try: - role = self._server.authentify(login, password) - except RpcError as err: - if err.exception == 'AuthenticationError': - logging.warning('Authentication error') - else: - logging.warning('Unknown error while authenticating: %s' % err) - return False - except Exception as err: - logging.debug('Unhandled exception: `%s`' % err) - else: - if role == 'hv': - self._handler = handlers.NodeHandler(self, - self._is_hv, - self._exec_cmd, - self._is_xen) - elif role == 'host': - self._handler = handlers.NodeHandler(self, - False, - self._exec_cmd, - self._is_xen) - else: - logging.warning('Bad role affected by server: %s' % role) - raise Exception() - - self._rpc.get_protocol(0).set_handler(self._handler) - self._scheduler_rearm() - return True - - def _scheduler_rearm(self): - ''' - ''' - self._scheduler_timer = Timer(5, self._scheduler_run) - self._scheduler_timer.start() - - def _scheduler_run(self): - ''' - ''' - with self._scheduler_mutex: - if not self._scheduler_stopped.is_set(): - self._handler.scheduler_run() - sleep(0.1) - self._scheduler_rearm() - - def get_server(self): - ''' - ''' - return self._server - - def get_rpc(self): - return self._rpc - - def get_handler(self): - ''' - ''' - return self._handler diff --git a/ccnode/common.py b/ccnode/common.py deleted file mode 100644 index 09644ff..0000000 --- a/ccnode/common.py +++ /dev/null @@ -1,359 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import re -import psutil -from subprocess import Popen, PIPE, STDOUT -from multiprocessing import cpu_count -from platform import platform, machine, system -from socket import gethostbyaddr, gethostname -from jobs import JobManager -from lvm import LVM -from drbd import DRBDPool - - -class Host(object): - ''' - Root class for all physical or virtual machines that CloudControl Node may - run on, or manage. It is not intended for direct usage by NodeHandler. - ''' - - -class LocalHost(Host): - ''' - Regular host with no hypervisor support, all methods defined here are - expected to provide information to the NodeHandler. - ''' - - ARCH = { - 'i386' : 'x86', - 'i486' : 'x86', - 'i586' : 'x86', - 'i686' : 'x86', - 'x86_64' : 'x64', - } - - def __init__(self): - ''' - ''' - super(LocalHost, self).__init__() - self.jobmgr = JobManager() - self.drbdpool = DRBDPool() - self.lvm = LVM() - - def scheduler_run(self): - ''' - ''' - pass - - def get_hw_serial(self): - ''' - ''' - serial = None - data = open('/sys/class/dmi/id/product_serial').read().strip() - if data: - serial = data - return serial - - def get_hw_vendor(self): - ''' - ''' - vendor = None - data = open('/sys/class/dmi/id/sys_vendor').read().strip() - if data: - vendor = data - return vendor - - def get_hw_product(self): - ''' - ''' - product = None - data = open('/sys/class/dmi/id/product_name').read().strip() - if data: - product = data - return product - - def get_hw_bios(self): - ''' - ''' - bios = '' - bios_ver = open('/sys/class/dmi/id/bios_version').read().strip() - bios_date = open('/sys/class/dmi/id/bios_date').read().strip() - if bios_ver: - bios += bios_ver - if bios_date: - bios += ' (%s)' % bios_date - if not bios: - bios = None - return bios - - def get_chassis_asset(self): - ''' - ''' - asset = None - data = open('/sys/class/dmi/id/chassis_asset_tag').read().strip() - if data: - asset = data - return asset - - def get_chassis_serial(self): - ''' - ''' - serial = None - data = open('/sys/class/dmi/id/chassis_serial').read().strip() - if data: - serial = data - return serial - - def get_name(self): - ''' - ''' - result = None - hostname = gethostname() - fqdn = gethostbyaddr(hostname)[0] - result = fqdn if fqdn else hostname - return result - - def get_uname(self): - ''' - ''' - uname = None - data = ' '.join(os.uname()) - if data: - uname = data - return uname - - def get_platform(self): - ''' - ''' - result = None - try: - p = platform() - if p: - result = p - except: - pass - return result - - def get_system(self): - ''' - ''' - result = None - try: - p = system() - if p: - result = p.lower() - except: - pass - return result - - def get_uptime(self): - ''' - ''' - uptime = None - try: - data = open("/proc/uptime").read().split() - if data: - uptime = int(float(data[0])) - except: - pass - return uptime - - def get_loadavg(self): - ''' - ''' - load = None - try: - data = ' '.join('%.2f' % load for load in os.getloadavg()) - if data: - load = data - except: - pass - return load - - def get_arch(self): - ''' - ''' - arch = None - try: - a = machine() - if a in self.ARCH: - arch = self.ARCH[a] - except: - pass - return arch - - def get_cpu(self): - ''' - ''' - cpucount = None - try: - data = cpu_count() - if data: - cpucount = data - except: - pass - return cpucount - - def get_cpu_usage(self): - ''' - ''' - usage = None - try: - data = '%.1f' % psutil.cpu_percent() - if data: - usage = data - except: - pass - return usage - - def get_mem(self): - ''' - ''' - mem = None - try: - data = psutil.avail_phymem() + psutil.used_phymem() - if data: - mem = data - except: - pass - return mem - - def get_mem_free(self): - ''' - ''' - free = None - try: - data = psutil.avail_phymem() - if data: - free = data - except: - pass - return free - - def get_mem_used(self): - ''' - ''' - used = None - try: - data = psutil.used_phymem() - if data: - used = data - except: - pass - return used - - def get_disks(self): - ''' - ''' - disks = {} - try: - re_pattern = re.compile(r'([sh]d[a-z]+)') - found = [bd for bd in os.listdir('/sys/block/') - if re_pattern.match(bd)] - for disk in found: - fullname = os.path.join('/sys/block', disk, 'size') - size = int(open(fullname).read()) - if size > 0: - disks[disk] = size * 512 - except: - pass - return disks - - def power_shutdown(self): - ''' - ''' - return self.execute('/sbin/shutdown -h -P 0') - - def power_off(self): - ''' - ''' - return self.execute('/sbin/shutdown -h -P -n 0') - - def power_reboot(self): - ''' - ''' - return self.execute('/sbin/shutdown -r -f 0') - - def power_force_reboot(self): - ''' - ''' - return self.execute('/sbin/shutdown -r -n 0') - - def execute(self, command): - ''' - ''' - output = None - try: - #FIXME: stop using shell=true and parse arguments with shlex.split() - data = Popen(command, - shell=True, - bufsize=-1, - stdin=PIPE, - stdout=PIPE, - stderr=STDOUT).communicate()[0] - if data: - output = data - except: - pass - return output - - -class Hypervisor(LocalHost): - ''' - ''' - - def __init__(self): - ''' - ''' - super(Hypervisor, self).__init__() - - def storage(self): - ''' - ''' - raise NotImplementedError - - def vm_list(self): - ''' - ''' - raise NotImplementedError - - def vm_list_running(self): - ''' - ''' - raise NotImplementedError - - def vm_list_stopped(self): - ''' - ''' - raise NotImplementedError - - def vm_list_paused(self): - ''' - ''' - raise NotImplementedError - - def vm_get(self, name): - ''' - ''' - raise NotImplementedError - - -class VM(Host): - ''' - ''' - - -class Storage(object): - ''' - ''' - - -class StoragePool(object): - ''' - ''' - - -class StorageVolume(object): - ''' - ''' diff --git a/ccnode/config.py b/ccnode/config.py new file mode 100644 index 0000000..a721a11 --- /dev/null +++ b/ccnode/config.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +import ConfigParser +from optparse import OptionParser +from . import __version__ + +ARGS_DEFAULTS = { + 'daemonize' : (bool, False), + 'stdout' : (bool, False), + 'config' : (str, '/etc/cc-node.conf'), + 'pidfile' : (str, '/var/run/cc-node.pid'), +} +FILE_DEFAULTS = { + 'address' : (str, ''), + 'port' : (int, 1984), + 'ssl' : (bool, True), + 'login' : (str, ''), + 'password' : (str, ''), + 'verbosity' : (int, 0), + 'virtualization' : (bool, True), + 'command_execution' : (bool, True), +} +SECURE_OPTIONS = [ + 'password', +] + + +class ConfigurationError(Exception): + + pass + + +class Configuration(object): + + def __init__(self, logger): + + self._logger = logger + + # populate default options + self._options = {} + for name, (vtype, value) in FILE_DEFAULTS.iteritems(): + self._options[name] = _ConfigValue(vtype, value) + for name, (vtype, value) in ARGS_DEFAULTS.iteritems(): + self._options[name] = _ConfigValue(vtype, value, locking=True) + + def parse_arguments(self): + + parser = OptionParser(version = '%%prog %s' % __version__) + + # --daemonize + parser.add_option('-d', + '--daemonize', + default = ARGS_DEFAULTS['daemonize'][1], + action = 'store_true', + help = 'run as daemon and write pid file') + # --stdout + parser.add_option('-s', + '--stdout', + action = 'store_true', + default = ARGS_DEFAULTS['stdout'][1], + help = 'log on standard output instead of syslog' + ' (when not running in daemon mode)') + # --config + parser.add_option('-c', + '--config', + default = ARGS_DEFAULTS['config'][1], + help = 'configuration file path (default: %default)') + # --pidfile + parser.add_option('-p', + '--pidfile', + default = ARGS_DEFAULTS['pidfile'][1], + help = 'pid file path for daemon mode (default: ' + '%default)') + + # parse command line + self._logger.info('reading command-line arguments') + options = parser.parse_args()[0] + + # apply arguments to current configuration + for name, value in options.__dict__.iteritems(): + if name in self._options: + self._options[name].update(value, unlock=True) + if name not in SECURE_OPTIONS: + self._logger.info(' * %s = %s' % (name, + self._options[name].value())) + + def parse_file(self): + + parser = ConfigParser.SafeConfigParser() + + # open configuration file + try: + self._logger.info('parsing configuration file') + parser.read(self._options['config'].value()) + options = dict(parser.items('node')) + except ConfigParser.Error as err: + raise ConfigurationError('failed to open or parse configuration' + ' file: %r' % err) + + # apply to current configuration + for name, value in options.iteritems(): + if name in self._options: + self._options[name].update(value, unlock=False) + if name not in SECURE_OPTIONS: + self._logger.info(' * %s = %s' % (name, + self._options[name].value())) + + def get(self, name): + + if name in self._options: + return self._options[name].value() + else: + return None + + def update(self, name, value): + + # TODO missing concurrency protection + if name in self._options: + self._options[name].update(value, unlock=False) + + +class _ConfigValue(object): + + def __init__(self, value_type, default_value, locking=False): + + self._type = value_type + self._default = default_value + self._value = default_value + self._locking = locking + + def __str__(self): + + return '' if self._value is None else str(self._value) + + def update(self, value, unlock=False): + + if not self._locking or (self._locking and unlock): + try: + # special case for boolean conversions + if self._type is bool and not isinstance(value, bool): + # any integer != 0 means True + if isinstance(value, int): + self._value = value != 0 + # a string meaning 'yes' + elif isinstance(value, str): + self._value = value.strip().lower() in ('yes', + 'y', '1', 'true', 'enable', + 'enabled', 'active', 'on') + # TODO rethink about a default value, or how to warn + else: + self._value = False + # or just let python do the conversion + else: + self._value = self._type(value) + except ValueError: + raise ConfigurationError('option has type `%r`, cannot' + ' convert from value `%s`' % (self._type, value)) + except TypeError: + raise ConfigurationError('option convertion to `%r` failed' + % (self._type)) + + def value(self): + + return self._value + + def default_value(self): + + return self._default + + def is_default(self): + + return self._value == self._default diff --git a/ccnode/drbd.py b/ccnode/drbd.py deleted file mode 100644 index 202fbdf..0000000 --- a/ccnode/drbd.py +++ /dev/null @@ -1,329 +0,0 @@ -# -*- coding: utf-8 -*- - -import xml.dom.minidom, os -from time import sleep -from errors import DRBDPoolError, DRBDError -from utils import RWLock, Exec, Enum - -class DRBDPool(object): - ''' - ''' - MAX_DEV = 16 - - _dev = dict([(i, None) for i in range(0, MAX_DEV)]) - _dev_mutex = RWLock() - - def _release_device(self, minor): - ''' - ''' - with self._dev_mutex.write: - self._dev[minor] = None - - def get_device(self, minor): - ''' - ''' - with self._dev_mutex.read: - if minor > 0 and minor < self.MAX_DEV: - if self._dev[minor] is not None: - return self._dev[minor] - else: - raise DRBDPoolError('device ID `%i` is not allocated' % - minor) - else: - raise DRBDPoolError('invalid device ID') - - def new_device(self): - ''' - ''' - with self._dev_mutex.write: - for minor, dev in self._dev.iteritems(): - if dev is None: - drbd = DRBD(self, minor) - self._dev[minor] = drbd - return drbd - - -class DRBD(object): - ''' - ''' - # tools binaries - BIN_PATH = '/sbin' - DRBDADM = os.path.join(BIN_PATH, 'drbdadm') - DRBDMETA = os.path.join(BIN_PATH, 'drbdmeta') - DRBDSETUP = os.path.join(BIN_PATH, 'drbdsetup') - MODPROBE = os.path.join(BIN_PATH, 'modprobe') - RMMOD = os.path.join(BIN_PATH, 'rmmod') - # connection states - CSTATES = Enum( - STANDALONE = "StandAlone", - DISCONNECTING = "Disconnecting", - UNCONNECTED = "Unconnected", - TIMEOUT = "Timeout", - BROKEN_PIPE = "BrokenPipe", - NETWORK_FAILURE = "NetworkFailure", - PROTOCOL_ERROR = "ProtocolError", - WF_CONNECTION = "WFConnection", - WF_REPORT_PARAMS = "WFReportParams", - TEAR_DOWN = "TearDown", - CONNECTED = "Connected", - STARTING_SYNC_S = "StartingSyncS", - STARTING_SYNC_T = "StartingSyncT", - WF_BITMAP_S = "WFBitMapS", - WF_BITMAP_T = "WFBitMapT", - WF_SYNC_UUID = "WFSyncUUID", - SYNC_SOURCE = "SyncSource", - SYNC_TARGET = "SyncTarget", - PAUSED_SYNC_S = "PausedSyncS", - PAUSED_SYNC_T = "PausedSyncT", - VERIFY_S = "VerifyS", - VERIFY_T = "VerifyT", - ) - # node roles - NROLES = Enum( - PRIMARY = "Primary", - SECONDARY = "Secondary", - UNKNOWN = "Unknown", - ) - # disk states - DSTATES = Enum( - DISKLESS = "Diskless", - ATTACHING = "Attaching", - FAILED = "Failed", - NEGOTIATING = "Negotiating", - INCONSISTENT = "Inconsistent", - OUTDATED = "Outdated", - UNKNOWN = "DUnknown", - CONSISTENT = "Consistent", - UP_TO_DATE = "UpToDate", - ) - - _mutex = RWLock() - - def __init__(self, manager, minor): - ''' - ''' - self._manager = manager - self._minor = int(minor) - self._path = '/dev/drbd%i' % self._minor - self._meta = None - self._dm_table = None - # load kernel driver, do not check for error - Exec.silent([self.RMMOD, 'drbd']) - Exec.silent([self.MODPROBE, 'drbd', 'minor_count=100', - 'usermode_helper=/bin/true']) - # check that binaries are available - for cmd in [self.DRBDADM, self.DRBDMETA, self.DRBDSETUP]: - if not os.access(cmd, os.F_OK): - raise DRBDError('failed to init DRBD device, support tool `%s`' - ' is not available' % cmd) - # check that device is not used - if self.status()['conn'] is not None: - print DRBDError('device minor `%i` is already configured, cannot' - ' use it' % self._minor) - - def destroy(self): - ''' - ''' - # bring down drbd device - Exec.silent([self.DRBDSETUP, - self._path, - 'detach']) - Exec.silent([self.DRBDSETUP, - self._path, - 'down']) - # release device from the pool - self._manager._release_device(self._minor) - # suicide - self._path = None - self._minor = None - - def get_minor(self): - ''' - ''' - return self._minor - - def get_port(self): - ''' - ''' - return 7788 + self._minor # FIXME magic number - - def get_path(self): - ''' - ''' - with self._mutex.read: - return self._path - - def status(self, nolock = False): - ''' - ''' - if not nolock: - self._mutex.read.acquire() - # fetch xml status - try: - rc, output = Exec.call([self.DRBDSETUP, self._path, 'status']) - except Exception as err: - if not nolock: - self._mutex.read.release() - raise err - else: - if rc: - raise DRBDError('failed to get device status') - # parse status - try: - xroot = xml.dom.minidom.parseString(output[0]) - xres = xroot.firstChild - status = {} - status['conn'] = self.CSTATES.get(xres.getAttribute('cs')) - status['disk'] = self.DSTATES.get(xres.getAttribute('ds1')) - status['rdisk'] = self.DSTATES.get(xres.getAttribute('ds2')) - status['role'] = self.NROLES.get(xres.getAttribute('ro1')) - status['rrole'] = self.NROLES.get(xres.getAttribute('ro2')) - if xres.hasAttribute('resynced_percent'): - status['percent'] = xres.getAttribute('resynced_percent') - else: - status['percent'] = None - return status - except Exception as err: - return {'conn' : None} - finally: - if not nolock: - self._mutex.read.release() - - def setup(self, sourcedev, metadev): - ''' - ''' - with self._mutex.write: - if self.status(nolock=True)['conn'] is None: - # wipe and init meta device - rc = Exec.silent([self.DRBDMETA, - '-f', - self._path, - 'v08', - metadev, - '0', - 'wipe-md']) - if rc != 0: - raise DRBDError('failed to clean meta device, maybe it is' - ' in use or does not exist') - rc = Exec.silent([self.DRBDMETA, - '-f', - self._path, - 'v08', - metadev, - '0', - 'create-md']) - if rc != 0: - raise DRBDError('failed to create meta device, maybe it is' - ' in use or does not exist') - else: - self._meta = metadev - # create the drbd disk - rc = Exec.silent([self.DRBDSETUP, - self._path, - 'disk', - sourcedev, - metadev, - '0', - '--create-device']) - if rc != 0: - raise DRBDError('failed to setup DRBD disk') - else: - raise DRBDError('cannot setup device, it is already configured') - - def connect(self, remote, remoteport, rate=50000): - ''' - ''' - with self._mutex.write: - # connect to remote node - rc = Exec.silent([self.DRBDSETUP, - self._path, - 'net', - '0.0.0.0:%i' % (self.get_port()), - '%s:%i' % (remote, remoteport), - 'C', - '-m', - '-S', - '10000000']) - if rc != 0: - raise DRBDError('failed to initiate connection to remote node,' - ' local port may already be in use') - # FIXME magic sleep seems to be mandatory - sleep(0.5) - # force sync rate - rc = Exec.silent([self.DRBDSETUP, - self._path, - 'syncer', - '-r', - str(rate)]) - if rc != 0: - raise DRBDError('failed to set sync rate') - - def disconnect(self): - ''' - ''' - # disconnect from remote node - Exec.silent([self.DRBDSETUP, - self._path, - 'disconnect']) - - def primary(self): - ''' - Switch the drbd to primary mode - ''' - with self._mutex.write: - rc = Exec.silent([self.DRBDSETUP, - self._path, - 'primary', - '-o']) - if rc != 0: - raise DRBDError('failed to switch to primary node mode') - - def secondary(self): - ''' - Switch the drbd to secondary mode - ''' - with self._mutex.write: - rc = Exec.silent([self.DRBDSETUP, - self._path, - 'secondary']) - if rc != 0: - raise DRBDError('failed to switch to secondary node mode') - - def wait_connection(self, timeout=300): - ''' - Wait for connection with the remote drbd node - ''' - with self._mutex.read: - sleep(0.5) #FIXME really needed ? - rc = Exec.silent([self.DRBDSETUP, - self._path, - 'wait-connect', - '-t', - str(timeout), - '-d', - str(timeout), - '-o', - str(timeout)]) - # FIXME magic sleep, seems to be mandatory - sleep(0.5) - try: - if rc != 0 or self.status()['conn'] != self.CSTATES.CONNECTED: - raise DRBDError('no connection after `%i` seconds' %timeout) - except Exception as err: - print err - - def wait_sync(self): - ''' - Wait for synchronization of the drbd device - ''' - with self._mutex.read: - rc = Exec.silent([self.DRBDSETUP, - self._path, - 'wait-sync']) - # FIXME magic sleep, seems to be mandatory - sleep(0.5) - status = self.status() - if (rc != 0 or not status or - ('disk' in status and - status['disk'] != self.DSTATES.UP_TO_DATE)): - raise DRBDError('synchronization failed') diff --git a/ccnode/errors.py b/ccnode/errors.py deleted file mode 100644 index 04c1767..0000000 --- a/ccnode/errors.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- - -class CCException(Exception): - ''' - ''' - def __init__(self, message, exception=None): - ''' - ''' - self._exception = exception - self._message = message - - def __str__(self): - ''' - ''' - if self._exception is not None: - return '[%s] %s' % (self._exception, self._message) - else: - return '%s' % self._message - - -# host - -class HostError(CCException): - ''' - ''' - pass - - -class HypervisorError(HostError): - ''' - ''' - pass - - -class VMError(HostError): - ''' - ''' - pass - - -# storage - -class StorageError(CCException): - ''' - ''' - pass - - -class StoragePoolError(StorageError): - ''' - ''' - pass - - -class StorageVolumeError(StorageError): - ''' - ''' - pass - - -# lvm errors - -class LVMError(CCException): - ''' - ''' - pass - -# drbd errors - -class DRBDPoolError(CCException): - ''' - ''' - pass - - -class DRBDError(CCException): - ''' - ''' - pass - -# job errors - -class JobManagerError(CCException): - ''' - ''' - pass - - -class JobError(CCException): - ''' - ''' - pass - - -class XferJobError(JobError): - ''' - ''' - pass - -class ReceiveFileJobError(XferJobError): - ''' - ''' - pass - -class SendFileJobError(XferJobError): - ''' - ''' - pass - -class DrbdCopyJobError(JobError): - ''' - ''' - pass - -class TCPTunnelJobError(JobError): - ''' - ''' - pass diff --git a/ccnode/handlers.py b/ccnode/handlers.py deleted file mode 100644 index ba7b66c..0000000 --- a/ccnode/handlers.py +++ /dev/null @@ -1,979 +0,0 @@ -# -*- coding: utf-8 -*- - -from logging import debug, error, warning, info -from fnmatch import fnmatchcase -from lvm import LVM -from time import sleep -from sjrpc.core import RpcError -from sjrpc.utils import RpcHandler -from errors import HostError, HypervisorError -from common import LocalHost -from jobs import ReceiveFileJob, SendFileJob, DrbdCopyJob, TCPTunnelJob -from __init__ import __version__ - - -#FIXME clean this up -_MOD_KVM = True -try: - import kvm -except ImportError: - _MOD_KVM = False -_MOD_XEN = True -try: - import xen -except ImportError: - _MOD_XEN = False - - -class NodeHandler(RpcHandler): - ''' - Main node handler that exports the host capabilities to the server. - ''' - - def __init__(self, connection, detect_hv=True, safe_mode=True, - force_xen=False): - ''' - ''' - super(NodeHandler, self).__init__() - self._connection = connection - self._safe_mode = safe_mode - - # create interface with host - self._host_handle = None - if detect_hv: - debug('Hypervisor detection in progress') - if _MOD_KVM and not force_xen: - debug('Initializing connection to the local KVM hypervisor') - self._host_handle = kvm.KvmHypervisor() - elif _MOD_XEN: - debug('Initializing connection to the local Xen hypervisor') - self._host_handle = xen.XenHypervisor() - - if self._host_handle is None: - debug('Hypervisor detection failed') - - if not detect_hv or self._host_handle is None: - debug('Hypervisor detection disabled, running as regular node') - self._host_handle = LocalHost() - - # methods that execute administrative commands, to be banned when - # running in safe mode - self.UNSAFE_METHODS = ['execute_command', 'shutdown'] - - # hypervisor tags - self.HV_TAG_MANDATORY = ['h'] - self.HV_TAG_MAP = { - # infinite TTL - 'version' : ( lambda o: True, - lambda o,t: str(__version__), - -1), - 'libvirtver': self._tag_map_direct('get_libvirt_version', -1), - 'htype' : self._tag_map_direct('get_hv_type', -1), - 'hserial' : self._tag_map_direct('get_hw_serial', -1), - 'hvendor' : self._tag_map_direct('get_hw_vendor', -1), - 'hmodel' : self._tag_map_direct('get_hw_product', -1), - 'arch' : self._tag_map_direct('get_arch', -1), - 'hvm' : self._tag_map_direct('get_hvm_available', -1), - 'cpu' : self._tag_map_direct('get_cpu', -1), - 'cpulogical': self._tag_map_direct('get_cpu_thread', -1), - 'chaserial' : self._tag_map_direct('get_chassis_serial', -1), - 'chaasset' : self._tag_map_direct('get_chassis_asset', -1), - # one day - 'hbios' : self._tag_map_direct('get_hw_bios', 24*3600), - 'hvver' : self._tag_map_direct('get_hv_version', 24*3600), - 'platform' : self._tag_map_direct('get_platform', 24*3600), - 'os' : self._tag_map_direct('get_system', 24*3600), - 'uname' : self._tag_map_direct('get_uname', 24*3600), - 'cpufreq' : self._tag_map_direct('get_cpu_frequency', 24*3600), - 'mem' : self._tag_map_direct('get_mem', 24*3600), - 'disk' : self._tag_map_keys('get_disks', 24*3600), - 'h' : self._tag_map_direct('get_name', 24*3600), - # one hour - # one minute - 'memfree' : self._tag_map_direct('get_mem_free', 60), - 'memused' : self._tag_map_direct('get_mem_used', 60), - 'sto' : ( lambda o: hasattr(o, 'storage'), - lambda o,t: ' '.join( - getattr(o, 'storage')().pool_list()), - 60), - # 5 seconds - 'uptime' : self._tag_map_direct('get_uptime', 5), - 'cpuuse' : self._tag_map_direct('get_cpu_usage', 5), - 'load' : self._tag_map_direct('get_loadavg', 5), - 'nvm' : self._tag_map_counter('vm_list', 5), - 'vmstarted' : self._tag_map_counter('vm_list_running', 5), - 'vmstopped' : self._tag_map_counter('vm_list_stopped', 5), - 'vmpaused' : self._tag_map_counter('vm_list_paused', 5), - 'rjobs' : ( lambda o: True, - lambda o,t: str(len( - self._host_handle.jobmgr.list()['running'])), - 5), - # 5 seconds - } - self.HV_TAG_GLOB = { - 'disk*' : self._tag_map_helper(self._helper_hv_disk, 24*3600), - 'sto*' : self._tag_map_helper(self._helper_hv_sto, 60), - } - - # guest VM tags - self.VM_TAG_MANDATORY = ['hv', 'h'] - self.VM_TAG_MAP = { - # infinite TTL - 'hv' : ( lambda o: hasattr(o, 'hypervisor'), - lambda o,t: o.hypervisor().get_name(), - -1), - 'htype' : ( lambda o: hasattr(o, 'hypervisor'), - lambda o,t: o.hypervisor().get_hv_type(), - -1), - 'arch' : self._tag_map_direct('get_arch', -1), - 'h' : self._tag_map_direct('get_name', -1), - # one day - # one hour - 'cpu' : self._tag_map_direct('get_cpu_core', 3600), - 'mem' : self._tag_map_direct('get_mem', 3600), - 'memmax' : self._tag_map_direct('get_mem_max', 3600), - 'vncport' : self._tag_map_direct('get_vnc_port', 3600), - # one minute - # 5 seconds - 'status' : ( lambda o: True, - lambda o,t: 'running' if o.is_active() - else 'paused' if o.is_paused() - else 'stopped', - 5), # FIXME crappy tag implementation - #'cpuuse' : self._tag_map_direct('get_cpu_usage'), - } - self.VM_TAG_GLOB = { - 'disk*' : self._tag_map_helper(self._helper_vm_disk, 3600), - 'nic*' : self._tag_map_helper(self._helper_vm_nic, 3600), - } - - # FIXME - self._register_vm = [] - - def __getitem__(self, name): - ''' - ''' - # filter the private members access - if name.startswith('_'): - raise KeyError('Remote name `%s` is private' % repr(name)) - # filter command execution methods - elif not self._safe_mode and name in self.UNSAFE_METHODS: - raise KeyError('Remote name `%s` is disabled by configuration' - % repr(name)) - else: - debug('Called %s.%s' % (self.__class__.__name__, name)) - return super(NodeHandler, self).__getitem__(name) - - def _tag_map_direct(self, method, ttl): - ''' - ''' - return ( lambda o: hasattr(o, method), - lambda o,t: getattr(o, method)(), - ttl) - - def _tag_map_counter(self, method, ttl): - ''' - ''' - return ( lambda o: hasattr(o, method), - lambda o,t: len(getattr(o, method)()), - ttl) - - def _tag_map_keys(self, method, ttl): - ''' - ''' - return ( lambda o: hasattr(o, method), - lambda o,t: ' '.join(getattr(o, method)().keys()), - ttl) - - def _tag_map_helper(self, helper, ttl): - ''' - ''' - return ( lambda o, resolve=False: helper(o, resolve=resolve), - lambda o, tag_name=None, resolve=False: - helper(o, tag_name=tag_name, resolve=resolve), - ttl) - - def _helper_hv_disk(self, hv, tag_name=None, resolve=True): - ''' - ''' - result = {} - disks = hv.get_disks() - if len(disks): - result['disk'] = ' '.join(disks.keys()) - for name, size in disks.iteritems(): - if size is not None: - result['disk%s_size' % name] = str(size) - if not result: - result = None - return result - - def _helper_hv_sto(self, hv, tag_name=None, resolve=True): - ''' - ''' - result = {} - if hasattr(hv, 'storage'): - pools = hv.storage().pool_list() - if len(pools): - result['sto'] = ' '.join(pools) - for pool_name in pools: - pool = hv.storage().pool_get(pool_name) - capa = pool.get_space_capacity() - if capa is not None: - result['sto%s_size' % pool_name] = str(capa) - free = pool.get_space_free() - if free is not None: - result['sto%s_free' % pool_name] = str(free) - used = pool.get_space_used() - if used is not None: - result['sto%s_used' % pool_name] = str(used) - vol = pool.volume_list() - if vol: - result['sto%s_vol' % pool_name] = ' '.join(vol) - if not result: - result = None - return result - - def _helper_vm_disk(self, vm, tag_name=None, resolve=True): - ''' - ''' - result = {} - volumes = vm.get_volumes() - if len(volumes): - result['disk'] = ' '.join([str(i) for i in range(0, len(volumes))]) - for vol_id, vol in enumerate(volumes): - name = vol.get_name() - if name: - result['disk%i_vol' % vol_id] = str(name) - pool = vol.get_pool() - if pool: - result['disk%i_pool' % vol_id] = str(pool.name()) - path = vol.get_path() - if path: - result['disk%i_path' % vol_id] = str(path) - capa = vol.get_space_capacity() - if capa is not None: - result['disk%i_size' % vol_id] = str(capa) - if not result: - result = None - return result - - def _helper_vm_nic(self, vm, tag_name=None, resolve=True): - ''' - ''' - result = {} - nics = vm.get_nics() - if len(nics): - result['nic'] = ' '.join([str(i) for i in range(0, len(nics))]) - for nic_id, nic in enumerate(nics): - mac = nic.get('mac') - if mac: - result['nic%i_mac' % nic_id] = str(mac) - model = nic.get('model') - if model: - result['nic%i_model' % nic_id] = str(model) - source = nic.get('source') - if source: - result['nic%i_source' % nic_id] = str(source) - if not result: - result = None - return result - - def scheduler_run(self): - ''' - ''' - # call host scheduler - if hasattr(self._host_handle, 'scheduler_run'): - self._host_handle.scheduler_run() - # (un)register sub nodes if this host has the capability - if hasattr(self._host_handle, 'vm_list'): - try: - vm_current = self._host_handle.vm_list() - for vm in vm_current: - if vm not in self._register_vm: - try: - info('registering vm `%s`' % vm) - self._connection.get_server().register(vm, 'vm') - except RpcError as e: - if e.exception == '#FIXME': - self._register_vm.append(vm) - else: - raise e - else: - self._register_vm.append(vm) - for vm in self._register_vm: - if vm not in vm_current: - try: - info('unregistering vm `%s`' % vm) - self._connection.get_server().unregister(vm) - except RpcError as e: - if e.exception == '#FIXME': - self._register_vm.remove(vm) - else: - raise e - else: - self._register_vm.remove(vm) - except Exception as e: - debug("REGISTER except `%s`:`%s`" % (repr(e), e)) - pass - - ################################## - # Tag query - ################################## - - def get_tags(self, tags=None, noresolve_tags=None): - ''' - ''' - result = {} - info('server requested tags=`%s` noresolve_tags=`%s`', tags, - noresolve_tags) - # build a single dict of tags, boolean means "resolve" - mytags = {} - if tags: - for t in tags: - mytags[t] = True - if noresolve_tags: - for t in noresolve_tags: - if t not in mytags: - mytags[t] = False - # return all tags if server does not request a subset - if not mytags: - # add simple tags - for t in self.HV_TAG_MAP.keys(): - mytags[t] = True - # add globbing tags - for pattern, handler in self.HV_TAG_GLOB.iteritems(): - try: - # helper is available on the current host - if handler[0](self._host_handle): - debug('host implements `%s`', pattern) - # get tags from helper - htags = handler[0](self._host_handle, resolve=False) - # append all tags - for t in htags: - mytags[t] = True - except Exception as err: - debug('error adding globbing tags `%r`:`%s`', err, err) - debug('no tag specified, expanded list to `%s`', mytags.keys()) - # add mandatory tags if missing in the list, or set noresolve - else: - for t in self.HV_TAG_MANDATORY: - if t not in mytags or not mytags[t]: - debug('add/correct mandatory tag `%s`', t) - mytags[t] = True - # query host - info('query host with tag list `%s`', mytags.keys()) - for tag, resolve in mytags.iteritems(): - # first, test tag name againts list of plain name - if tag in self.HV_TAG_MAP: - debug('A1: plain mapping found for tag `%s`', tag) - try: - if self.HV_TAG_MAP[tag][0](self._host_handle): - debug('B1: tag `%s` is available on host', tag) - result[tag] = {} - result[tag]['ttl'] = self.HV_TAG_MAP[tag][2] - if resolve: - debug('C1: now resolving tag `%s`', tag) - # fetch tag data - q = self.HV_TAG_MAP[tag][1](self._host_handle, tag) - debug('D1: host returned `%s`=`%s`', tag, q) - if q is not None: - # append tag data - result[tag]['value'] = str(q) - else: - debug('E1: I wont return `%s`=`None`', tag) - else: - debug('tag `%s` is not implemented !!!', tag) - except Exception as err: - debug('error with tag mapping `%r`:`%s`', err, err) - # if no direct tag mapping exists, test name against globbing - # a list - else: - debug('A2: searching for `%s` in globbing tags', tag) - # iter on globbing patterns, and get helper references - # process the first globbing tag that match then exit bcause - # there should not exist two globbing pattern matching - # one tag, ideally - for pattern, handler in self.HV_TAG_GLOB.iteritems(): - try: - # helper is available on the current host - if handler[0](self._host_handle, tag): - debug('B2: testing pattern `%s`', pattern) - if fnmatchcase(tag, pattern): - debug('C2: processing tag `%s` with pattern `%s`', - tag, pattern) - # get tags from helper - htags = handler[1](self._host_handle, tag) - # FIXME intead of extracting one tag, try not - # to build the whole list. Maybe it's too - # difficult and not worth to implement - if tag in htags: - debug('D2: found tag in helper result with' - ' value `%s`', htags[tag]) - result[tag] = {} - result[tag]['ttl'] = handler[2] - if resolve and htags[tag] is not None: - result[tag]['value'] = str(htags[tag]) - - break - except Exception as err: - debug('error with globbing tag `%r`:`%s`', err, err) - return result - - def _sub_tag_list(self, sub_obj): - ''' - ''' - result = [] - # add simple tags - result.extend(self.VM_TAG_MAP.keys()) - # add globbing tags - for pattern, handler in self.VM_TAG_GLOB.iteritems(): - try: - # helper is available on the current host - if handler[0](sub_obj): - debug('sub node implements `%s`' % pattern) - # get tags from helper - htags = handler[0](sub_obj, resolve=False) - debug('handler provides `%s`' % htags) - # append all tags - for t in htags.keys(): - result.append(t) - except Exception as err: - debug('error while listing sub node tags `%r`:`%s`', err, err) - return result - - def sub_tags(self, sub_id, tags=None, noresolve_tags=None): - ''' - ''' - info('server requested tags for `%s` with tags=`%s` noresolve_tags=`%s`' - , sub_id, tags, noresolve_tags) - if sub_id not in self._host_handle.vm_list(): - debug('sub node `%s` is unknown !', sub_id) - raise HypervisorError('sub node `%s` is unknown' % sub_id) - else: - # open a wrapper to the VM - debug('fetching vm data for `%s`', sub_id) - sub = self._host_handle.vm_get(sub_id) - # build a single list of tags - debug('server requested tags `%s` + `%s`', tags, noresolve_tags) - available_tags = self._sub_tag_list(sub) - mytags = {} - # return all resolved tags case - if tags is None and noresolve_tags is None: - for t in available_tags: - mytags[t] = True - elif tags is None or noresolve_tags is None: - if tags is None: - for t in available_tags: - mytags[t] = True - for t in noresolve_tags: - mytags[t] = False - else: - for t in available_tags: - mytags[t] = False - for t in tags: - mytags[t] = True - else: - for t in noresolve_tags: - mytags[t] = False - for t in tags: - mytags[t] = True - debug('expanded list to `%s`', mytags.keys()) - # add mandatory tags if missing in the list, or set noresolve - for t in self.VM_TAG_MANDATORY: - if t not in mytags or not mytags[t]: - debug('add/correct mandatory tag `%s`' % t) - mytags[t] = True - # query the subnode - result = {} - try: - # query the VM with each tag - for tag, resolve in mytags.iteritems(): - # first, search tag in plain mappings - if tag in self.VM_TAG_MAP: - debug('A1: plain mapping found for tag `%s`', tag) - # proceed if tag can be resolved on this VM - try: - if self.VM_TAG_MAP[tag][0](sub): - result[tag] = {} - result[tag]['ttl'] = self.VM_TAG_MAP[tag][2] - if resolve: - debug('B1: resolving tag `%s`', tag) - # call the wrapper mapping lambda - q = self.VM_TAG_MAP[tag][1](sub, tag) - debug('C1: tag query returned `%s`=`%s`', tag, q) - if q is not None: - if resolve: - result[tag]['value'] = str(q) - else: - warning('D1: I wont return `%s`=`None`', tag) - except Exception as err: - debug('error with plain mapping `%r`:`%s`', err, err) - # no tag mapping exist, test name against the globbing list - else: - debug('A2: searching for `%s` in globbing tags', tag) - # iter on globbing patterns, and get helper references - # process the first globbing tag that match then exit - # because there should not exist two globbing pattern - # matching one tag, ideally - for pattern, handler in self.VM_TAG_GLOB.iteritems(): - try: - # helper is available on the current VM - if handler[0](sub, tag): - if fnmatchcase(tag, pattern): - debug('B2: processing tag `%s` with pattern' - ' `%s`' % (tag, pattern)) - # get tags from helper - htags = handler[1](sub, tag) - # FIXME intead of extracting one tag, try - # not to build the whole list. Maybe it's - # too difficult and not worth implementing - if tag in htags: - debug('C2: found tag in helper result with' - ' value `%s`' % htags[tag]) - result[tag] = {} - result[tag]['ttl'] = handler[2] - if resolve: - result[tag]['value'] = str( - htags[tag]) - break - except Exception as err: - debug('error with globbing tag `%r`:`%s`', err, err) - except Exception as err: - debug('global error `%r`:`%s`', err, err) - return result - - ################################## - # Host control - ################################## - - def node_shutdown(self, reboot=True, gracefull=True): - ''' - ''' - info('server requested shutdown of local host with options ' - 'reboot=`%s` gracefull=`%s`', reboot, gracefull) - if reboot: - method = 'power_reboot' if gracefull else 'power_force_reboot' - else: - method = 'power_shutdown' if gracefull else 'power_off' - if hasattr(self._host_handle, method): - result = getattr(self._host_handle, method)() - info('in progress... action returned `%s`', result) - return result - else: - info('unable to proceed, this feature is not available') - raise NotImplementedError('host handler has no method `%s`' %method) - - def execute_command(self, command): - ''' - ''' - info('starting execution of `%s`' % command) - output = self._host_handle.execute(command) - info('finished execution of `%s`' % command) - return output - - ################################## - # VM management - ################################## - - ### - # Definition and migration of domains - - def vm_define(self, data, format='xml'): - ''' - ''' - debug('request for creation of a new VM with data=`%s`', data) - name = self._host_handle.vm_define(data) - debug('new VM has name `%s`', name) - return name - - def vm_undefine(self, name): - ''' - ''' - debug('request for deletion of VM definition named `%s`', name) - vm = self._host_handle.vm_get(name) - vm.undefine() - debug('succesfully undefined VM `%s`', name) - - def vm_export(self, name, format='xml'): - ''' - ''' - debug('request for definition data of VM `%s`', name) - vm = self._host_handle.vm_get(name) - return vm.get_config() - - ### - # Life cycle control - - def vm_stop(self, vm_names=None, force=False): - ''' - ''' - info('server requested stop of `%s`, force is `%s`', vm_names, force) - if vm_names is None: - vm_names = self._host_handle.vm_list_running() - debug('no vm specified, expanded list to `%s`', vm_names) - elif not isinstance(vm_names, list): - vm_names = [vm_names] - for vm_name in vm_names: - try: - debug('fetching vm data for `%s`', vm_name) - vm = self._host_handle.vm_get(vm_name) - if force: - debug('powering off `%s`', vm_name) - vm.power_off() - else: - debug('shutdown requested for `%s`', vm_name) - vm.power_shutdown() - except: - pass - - def vm_start(self, vm_names=None): - ''' - ''' - info('server requested start of `%s`', vm_names) - if vm_names is None: - vm_names = self._host_handle.vm_list_stopped() - debug('no vm specified, expanded list to `%s`', vm_names) - elif not isinstance(vm_names, list): - vm_names = [vm_names] - for vm_name in vm_names: - try: - debug('fetching vm data for `%s`', vm_name) - vm = self._host_handle.vm_get(vm_name) - debug('powering on `%s`', vm_name) - vm.power_on() - except: - pass - - def vm_suspend(self, vm_names=None): - ''' - ''' - info('server requested suspend of `%s`', vm_names) - if vm_names is None: - vm_names = self._host_handle.vm_list_running() - debug('no vm specified, expanded list to `%s`',vm_names) - elif not isinstance(vm_names, list): - vm_names = [vm_names] - for vm_name in vm_names: - try: - debug('fetching vm data for `%s`', vm_name) - vm = self._host_handle.vm_get(vm_name) - debug('pause execution of `%s`', vm_name) - vm.power_suspend() - except: - pass - - def vm_resume(self, vm_names=None): - ''' - ''' - info('server requested resume of `%s`', vm_names) - if vm_names is None: - vm_names = self._host_handle.vm_list_paused() - debug('no vm specified, expanded list to `%s`', vm_names) - elif not isinstance(vm_names, list): - vm_names = [vm_names] - for vm_name in vm_names: - try: - debug('fetching vm data for `%s`', vm_name) - vm = self._host_handle.vm_get(vm_name) - debug('resume execution of `%s`', vm_name) - vm.power_resume() - except: - pass - - ### - # Migration and helpers - - def tun_setup(self, local=True): - ''' - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # create the job - job = TCPTunnelJob(jobmgr) - job.prepare() - job.start_now() - # start listening - job.tunnel_listen(local) - # build a ressource - return { - 'jid' : job.get_id(), - 'key' : 'FIXME', - 'port' : job.tunnel_get_server_port(), - } - - def tun_connect(self, res, remote_res, remote_ip): - ''' - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # get the job - job = jobmgr.get_job(res['jid']) - # connect to the remote endpoint - job.tunnel_connect((remote_ip, remote_res['port'])) - - def tun_connect_hv(self, res, migration=False): - ''' - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # get the job - job = jobmgr.get_job(res['jid']) - # choose connection endpoint - if self._host_handle.get_hv_type() == 'xen' and migration: - # FIXME magic values - ip = '127.0.0.1' - port = 8002 - endpoint = (ip, port) - else: - # FIXME magic values - endpoint = '/var/run/libvirt/libvirt-sock' - # connect to libvirt/hv - job.tunnel_connect(endpoint) - - def tun_destroy(self, res): - ''' - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # stop the tunnel - jobmgr.cancel(res['jid']) - - def vm_migrate_tunneled(self, name, tun_res, migtun_res): - ''' - ''' - debug('request for tunneled migration of VM `%s`', name) - # local listenning tunnels ports - tun_port = tun_res.get('port') - migtun_port = migtun_res.get('port') - debug('ports are: tunnel=`%d` migration=`%d`', tun_port, migtun_port) - # acquire vm object - vm = self._host_handle.vm_get(name) - # migrate - vm.migrate(('127.0.0.1', tun_port), ('127.0.0.1', migtun_port)) - - - ################################## - # Storage control - ################################## - - ### - # Volume management - - def vol_create(self, pool, name, size): - ''' - ''' - size = int(size) - if hasattr(self._host_handle, 'storage'): - pool = self._host_handle.storage().pool_get(pool) - pool.volume_create(name, size) - else: - raise NotImplementedError('host handler has no storage support') - - def vol_delete(self, pool, name, wipe=False): - ''' - ''' - if hasattr(self._host_handle, 'storage'): - pool = self._host_handle.storage().pool_get(pool) - vol = pool.volume_get(name) - if wipe: - vol.wipe() - vol.delete() - else: - raise NotImplementedError('host handler has no storage support') - - def vol_copy(self, pool, name, dest_pool, dest_name): - raise NotImplementedError() - - ### - # Basic network copy - - def vol_export(self, pool, name, raddr, rport): - ''' - ''' - rport = int(rport) - if hasattr(self._host_handle, 'storage'): - # facilities - sto = self._host_handle.storage() - jmgr = self._host_handle.jobmgr - # get device path - vol_path = sto.pool_get(pool).volume_get(name).get_path() - # create job - job = SendFileJob(jmgr, vol_path, raddr, rport) - job.start_now() - jid = job.get_id() - # wait for job completion - while self._host_handle.jobmgr.is_job_running(jid): - sleep(2) - # return job report - res = {} - res['id'] = jid - res['log'] = job.get_log() - if jmgr.is_job_finished(jid): - res['checksum'] = job.get_checksum() - return res - else: - raise NotImplementedError('host handler has no storage support') - - def vol_import(self, pool, name): - ''' - ''' - if hasattr(self._host_handle, 'storage'): - # facilities - sto = self._host_handle.storage() - # get device path - vol_path = sto.pool_get(pool).volume_get(name).get_path() - # create job - job = ReceiveFileJob(self._host_handle.jobmgr, vol_path) - # do the preparation immediately, so we can fetch the port number - # before the job starts - job.prepare() - job.start_now() - # return job info - res = {} - res['id'] = job.get_id() - res['port'] = job.get_port() - return res - else: - raise NotImplementedError('host handler has no storage support') - - def vol_import_wait(self, jid): - ''' - ''' - jid = int(jid) - jmgr = self._host_handle.jobmgr - # wait for job completion - while jmgr.is_job_running(jid): - sleep(2) - # get the job - job = jmgr.get_job(jid) - # return job report - res = {} - res['id'] = job.get_id() - res['log'] = job.get_log() - if jmgr.is_job_finished(job.get_id()): - res['checksum'] = job.get_checksum() - return res - - def vol_import_cancel(self, jid): - ''' - ''' - jid = int(jid) - # facilities - jmgr = self._host_handle.jobmgr - # get the job - job = jmgr.get_job(jid) - # check job type - if job.get_type() != 'ReceiveFileJob': - raise HostError('this job ID does not match a volume import') - # cancel it - jmgr.cancel(jid) - - ### - # Live network copy - - def drbd_setup(self, pool, name): - ''' - ''' - if hasattr(self._host_handle, 'storage'): - # facilities - sto = self._host_handle.storage() - lvm = self._host_handle.lvm - jobmgr = self._host_handle.jobmgr - drbdpool = self._host_handle.drbdpool - # get pool and volume handles - vol_pool = sto.pool_get(pool) - vol = vol_pool.volume_get(name) - # stop if it's not an LVM volume - if vol_pool.get_source_format() != 'lvm2': - raise HostError('not a LVM2 volume, cannot use DRBD') - # fetch LV/VG info - vg_name = vol_pool.get_source_name() - lv_name = vol.get_name() - # create job - job = DrbdCopyJob(jobmgr, lvm, drbdpool, vg_name, lv_name) - job.start_now() - return { - 'jid' : job.get_id(), - 'port' : job.drbd_get_port(), - } - else: - raise NotImplementedError('host handler has no storage support') - - def drbd_connect(self, res, remote_res, remote_ip): - ''' - Connect the local live copy engine to the designated remote host that - will push or pull data. - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # get the job - job = jobmgr.get_job(res['jid']) - # connect node - job.drbd_connect(remote_ip, remote_res['port']) - - def drbd_role(self, res, primary): - ''' - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # get the job - job = jobmgr.get_job(res['jid']) - # change role - job.drbd_role(primary=primary) - - def drbd_takeover(self, res, state): - ''' - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # get the job - job = jobmgr.get_job(res['jid']) - # hijack the VM disk DM - job.drbd_takeover(state) - - def drbd_sync_status(self, res): - ''' - Get status information about a running sync - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # get the job - job = jobmgr.get_job(res['jid']) - # get drbd status - status = job.drbd_status() - # return status - return { - 'done' : status['disk'] == job._drbd.DSTATES.UP_TO_DATE, - 'completion' : status['percent'], - 'speed' : None, - } - - def drbd_shutdown(self, res): - ''' - Close connection and destroy all ressources allocated for a live copy. - ''' - # facilities - jobmgr = self._host_handle.jobmgr - # cancel the job so it can clean itself - # FIXME: check type, we don't want to cancel anything - jobmgr.cancel(res['jid']) - - ################################## - # Job management - ################################## - - def job_list(self): - ''' - List all existing jobs sorted by state, with ID and type. - ''' - return self._host_handle.jobmgr.list() - - def job_log(self, jid): - ''' - Get log messages of a given job (human friendly text string with - carriage return). - ''' - # get the job - job = self._host_handle.jobmgr.get_job(jid) - # return log string - return job.get_log() diff --git a/ccnode/jobs.py b/ccnode/jobs.py index 7dfd405..c157bda 100644 --- a/ccnode/jobs.py +++ b/ccnode/jobs.py @@ -1,1127 +1,32 @@ # -*- coding: utf-8 -*- - -import os, time, socket, select -from hashlib import md5 -from datetime import datetime -from threading import Lock, RLock, Thread, Event -from logging import debug -from utils import RWLock -from drbd import DRBD -from lvm import LVM -from errors import (JobManagerError, JobError, XferJobError, - ReceiveFileJobError, SendFileJobError, DrbdCopyJobError, LVMError, - DRBDError, TCPTunnelJobError) - -#fixme -import traceback, sys - -class JobManager(object): - ''' - ''' - MAX_JID = 65535 - - def __init__(self): - ''' - ''' - super(JobManager, self).__init__() - self._mutex = RLock() - self._next_jid = 1 - # job objects indexed by job ID - self._jobs = {} - # new jobs IDs - self._jobs_pending = [] - # running jobs IDs - self._jobs_running = [] - # dead jobs IDs - self._jobs_crashed = [] - self._jobs_finished = [] - - def _job_crashed(self, jid): - debug('JobManager._job_crashed: id=`%i`' % jid) - with self._mutex: - # job should exist - job = self.get_job(jid) - # move job to the crashed queue - self._jobs_crashed.append(jid) - # set the cancelled flag to block pending commands - job._cancelled = True - # clean any queue that may contain the job - for queue in [self._jobs_pending, self._jobs_running, - self._jobs_finished]: - if jid in queue: - queue.remove(jid) - # FIXME put at the end because it may raise errors - # execute job self cleanup - job._cleanup() - - def _job_finished(self, jid): - debug('JobManager._job_finished: id=`%i`' % jid) - with self._mutex: - # job should exist - job = self.get_job(jid) - # move job to the finished queue - self._jobs_finished.append(jid) - # remove from running queue - self._jobs_running.remove(jid) - # FIXME put at the end because it may raise errors - # execute job self cleanup - job._cleanup() - - def run(self): - ''' - ''' - # FIXME - while True: - pass - - def append(self, job): - ''' - ''' - with self._mutex: - jid = self._next_jid - self._next_jid += 1 - self._jobs[jid] = job - self._jobs_pending.append(jid) - debug('JobManager.append: new job pending with id `%i`', jid) - return jid - - def cancel(self, jid): - ''' - ''' - with self._mutex: - # job should exist - job = self.get_job(jid) - # job should be running - if jid not in self._jobs_running: - raise JobManagerError('job `%i` is not running' % jid) - # ask the job to stop it's execution - self._jobs[jid]._cancelled = True - self._jobs[jid]._cancel() - # do NOT execute _cleanup() here !!! - - def list(self): - ''' - ''' - with self._mutex: - pending = [] - running = [] - crashed = [] - finished = [] - orphaned = [] - for jid, job in self._jobs.iteritems(): - s = (jid, job.get_type()) - if jid in self._jobs_pending: - pending.append(s) - elif jid in self._jobs_running: - running.append(s) - elif jid in self._jobs_crashed: - crashed.append(s) - elif jid in self._jobs_finished: - finished.append(s) - else: - orphaned.append(s) - return { - 'pending' : pending, - 'running' : running, - 'crashed' : crashed, - 'finished' : finished, - 'orphaned' : orphaned, - } - - def schedule_immediate(self, jid): - ''' - ''' - debug('JobManager.schedule_immediate: id=`%i`', jid) - with self._mutex: - # job should exist - job = self.get_job(jid) - # job should be pending execution - if jid not in self._jobs_pending: - raise JobManagerError('job `%i` not prepared for execution'%jid) - # execute job - self._jobs_running.append(jid) - self._jobs_pending.remove(jid) - job.daemon = True - job.start() - - def is_job(self, jid): - ''' - ''' - with self._mutex: - return jid in self._jobs - - def get_job(self, jid): - ''' - ''' - with self._mutex: - if self.is_job(jid): - return self._jobs[jid] - else: - raise JobManagerError('unknown job ID `%i`' % jid) - - def is_job_pending(self, jid): - ''' - ''' - with self._mutex: - if not self.is_job(jid): - raise JobManagerError('unknown job ID `%i`' % jid) - return jid in self._jobs_pending - - def is_job_running(self, jid): - ''' - ''' - with self._mutex: - if not self.is_job(jid): - raise JobManagerError('unknown job ID `%i`' % jid) - return jid in self._jobs_running - - def is_job_crashed(self, jid): - ''' - ''' - with self._mutex: - if not self.is_job(jid): - raise JobManagerError('unknown job ID `%i`' % jid) - return jid in self._jobs_crashed - - def is_job_finished(self, jid): - ''' - ''' - with self._mutex: - if not self.is_job(jid): - raise JobManagerError('unknown job ID `%i`' % jid) - return jid in self._jobs_finished - - def is_job_cancelled(self, jid): - ''' - ''' - with self._mutex: - job = self.get_job(jid) - return job._cancelled is True - - -class JobLog(): - ''' - ''' - def __init__(self): - ''' - ''' - self._items = [] - self._mutex = RLock() - - def __str__(self): - ''' - ''' - res = "" - for tup in self._items: - res += "[%s] %s\n" % (datetime.fromtimestamp(tup[0]).strftime( - "%Y-%m-%d %H:%M:%S"), tup[1]) - return res - - def append(self, message): - ''' - ''' - # FIXME add limit to buffer - debug('JobLog.append: %s', message) - with self._mutex: - self._items.append((time.time(), message)) - - -class BaseJob(Thread, object): - ''' - ''' - def __init__(self, manager): - ''' - ''' - super(BaseJob, self).__init__() - self._manager = manager - self._log = JobLog() - # job state - self._type = self.__class__.__name__ - self._validated = None - self._ready_run = False - self._cancelled = False - # store job in manager - self._jid = self._manager.append(self) - self.name = '%s-%i' % (self._type, self._jid) - self._log.append('stored with id `%i` and type `%s`' % (self._jid, - self._type)) - - def _validate(self): - ''' - To be implemented in derivate class - ''' - return False - - def _prepare(self): - ''' - To be implemented in derivate class - ''' - return False - - def _cancel(self): - ''' - To be implemented in derivate class, or not - ''' - pass - - def _cleanup(self): - ''' - To be implemented in derivate class, or not - ''' - pass - - def _job(self): - ''' - To be implemented in derivate class - ''' - self._log.append('nothing to do') - raise JobError('empty job') - - def is_ready(self): - ''' - ''' - return self._ready_run is True - - def get_id(self): - ''' - ''' - return self._jid - - def get_log(self): - ''' - ''' - return str(self._log) - - def get_type(self): - ''' - ''' - return self._type - - def start_now(self): - ''' - ''' - self._manager.schedule_immediate(self._jid) - - def run(self): - ''' - ''' - try: - if not self._ready_run: - self.prepare() - self._log.append("running") - self._job() - self._log.append("finished") - self._manager._job_finished(self._jid) - except Exception as err: - self._log.append("*** CRASHED: `%r`: `%s`" % (err, err)) - self._manager._job_crashed(self._jid) - - def prepare(self): - ''' - ''' - if not self._ready_run: - # run validation if not done yet - if self._validated is None: - try: - self._validated = self._validate() - except: - self._validated = False - finally: - # do not continue if validation fails - if not self._validate: - self._log.append('validation failed') - raise JobError('validation failed') - # run validation in derivated Job class - try: - ready = self._prepare() - except: - ready = False - finally: - self._ready_run = ready - # do not continue if preparation fails - if not ready: - self._log.append('preparation failed') - raise JobError('preparation failed') - else: - self._log.append('ready for execution') - - -class XferJob(BaseJob): - ''' - ''' - TYPES = [ - 'SendFileJob', - 'ReceiveFileJob', - ] - - # tcp port range used for transferts - TCP_PORT_MIN = 42000 - TCP_PORT_MAX = 43999 - - def __init__(self, manager, path): - ''' - ''' - self._path = path - self._md5 = md5() - self._checksum = None - super(XferJob, self).__init__(manager) - - def _validate(self): - ''' - ''' - return self._type in self.TYPES - - def get_checksum(self): - return self._checksum - - -#TODO rewrite with the _cancel() and _cleanup() methods -class SendFileJob(XferJob): - ''' - ''' - def __init__(self, manager, path, remote_host, remote_port): - ''' - ''' - self._host = remote_host - self._local_port = int(remote_port) - super(SendFileJob, self).__init__(manager, path) - - def _validate(self): - ''' - ''' - return ( super(SendFileJob, self)._validate() - and os.path.isfile(self._path) - and os.access(self._path, os.F_OK | os.R_OK ) - and self._local_port >= self.TCP_PORT_MIN - and self._local_port <= self.TCP_PORT_MAX - and isinstance(self._host, str)) - - def _prepare(self): - ''' - ''' - # create socket - self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - return bool(self._sock) - - def _job(self): - ''' - ''' - fd = None - try: - # try to connect - self._log.append('connecting to `%s`:`%i`' % (self._host, - self._local_port)) - self._sock.settimeout(10) - retry = 0 - connected = False - while retry < 6 and not connected and not self._cancelled: - retry += 1 - try: - self._sock.connect((self._host, self._local_port)) - except socket.timeout: - pass - except socket.error: - time.sleep(1) - pass - else: - connected = True - if self._cancelled: - self._log.append('stopped connect attempt (job was cancelled)') - elif connected: - self._log.append('connected') - # open file - self._log.append('opening source file `%s`' % self._path) - fd = open(self._path, 'rb') - # read and send file - self._log.append('sending data') - while True and not self._cancelled: - buf = fd.read(4096) - if not buf: - break - self._sock.send(buf) - self._md5.update(buf) - # report transfert completion - if self._cancelled: - self._log.append('transfert aborted (job was cancelled)') - else: - self._log.append('no more data to send') - self._checksum = self._md5.hexdigest() - self._md5 = None - self._log.append('transfered data checksum = `%s`' % - self._checksum) - else: - self._log.append('remote host did not accept connection, ' - 'aborting emission') - except: - #traceback.print_exc(file=sys.stdout) - raise SendFileJobError('FIXME error reporting') - finally: - # close socket and files anyway - try: - self._sock.close() - except: - pass - try: - fd.close() - except: - pass - - -#TODO rewrite with the _cancel() and _cleanup() methods -class ReceiveFileJob(XferJob): - ''' - ''' - _ports_used_local = [] - _ports_used_mutex = RWLock() - - def __init__(self, manager, path, create=False): - ''' - ''' - self._create = create - self._local_port = None - super(ReceiveFileJob, self).__init__(manager, path) - - def _validate(self): - ''' - ''' - return ( super(ReceiveFileJob, self)._validate() and - ( - ( not self._create and - os.path.isfile(self._path) and - os.access(self._path, os.F_OK | os.W_OK )) - or - ( self._create and - os.path.isdir(os.path.dirname(self._path)) and - os.access(os.path.dirname(self._path), os.F_OK |os.W_OK) - ) - )) - - def _prepare(self): - ''' - ''' - # create socket - self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # bind socket to a port - for port in range(self.TCP_PORT_MIN, self.TCP_PORT_MAX): - # find a port to bind to - try: - with self._ports_used_mutex.write: - if port not in self._ports_used_local: - self._sock.bind(('', port)) - self._local_port = port - self._ports_used_local.append(port) - break - except socket.error: - pass - if not self._local_port: - msg = 'no port to listen to' - self._log.append(msg) - raise ReceiveFileJobError(msg) - return True - - def _job(self): - ''' - ''' - fd = None - try: - self._sock.listen(1) - self._sock.settimeout(1) - # wait for connection - self._log.append('waiting for connection on port `%i`' % self._local_port) - retry = 0 - conn = None - while retry < 60 and not conn and not self._cancelled: - retry += 1 - try: - conn, addr = self._sock.accept() - except socket.timeout as err: - pass - if self._cancelled: - self._log.append('stopped waiting (job was cancelled)') - if conn: - self._log.append('client connected from `%s`') - # open file - self._log.append('opening destination `%s`' % self._path) - fd = open(self._path, 'wb') - # write to the file - self._log.append('receiving data') - while True and not self._cancelled: - buf = conn.recv(4096) - if not buf: - break - fd.write(buf) - self._md5.update(buf) - # report transfert completion - if self._cancelled: - self._log.append('transfert aborted (job was cancelled)') - else: - self._log.append('remote host closed connection') - self._checksum = self._md5.hexdigest() - self._md5 = None - self._log.append('transfered data checksum = `%s`' % - self._checksum) - else: - self._log.append('no connection received, aborting reception') - except: - #traceback.print_exc(file=sys.stdout) - raise ReceiveFileJobError('FIXME error reporting') - finally: - # close socket and files anyway - try: - self._sock.close() - except: - pass - with self._ports_used_mutex.write: - if self._local_port in self._ports_used_local: - self._ports_used_local.remove(self._local_port) - try: - fd.close() - except: - pass - - def get_port(self): - ''' - ''' - return int(self._local_port) - - -class DrbdCopyJob(BaseJob): - ''' - ''' - def __init__(self, manager, lvm, drbd_pool, source_vg, source_lv): - ''' - ''' - self._mutex_step = Lock() - self._lvm = lvm - self._pool = drbd_pool - self._source_vgname = source_vg - self._source_lvname = source_lv - super(DrbdCopyJob, self).__init__(manager) - self._drbd = self._pool.new_device() - # checkpoint events - self._event_connected_ready = Event() - self._event_connected = Event() - self._event_rolechosen_ready = Event() - self._event_rolechosen = Event() - self._event_sleep_ready = Event() - self._event_sleep = Event() - - def _cancel(self): - ''' - ''' - # release some pending calls - self._event_connected_ready.set() - self._event_connected.set() - self._event_rolechosen_ready.set() - self._event_rolechosen.set() - self._event_sleep_ready.set() - self._event_sleep.set() - # disconnect the node - self._drbd.disconnect() - - def _cleanup(self): - ''' - ''' - # destroy drbd device - try: - self._log.append('destroying the drbd device') - self._drbd.destroy() - except Exception as err: - self._log.append('failed to destroy the drbd device') - else: - self._log.append('successfully destroyed the drbd device') - - # we wait for any potential job step to finish - # should be quick after the drbd destruction (kills all waits) - with self._mutex_step: - # destroy drbd meta - try: - self._log.append('removing the drbd meta volume') - self._lvm.get_vg(self._source_vgname).remove(self._meta_lvname) - except Exception as err: - self._log.append('failed to remove the drbd meta volume') - else: - self._log.append('successfully removed the drbd meta volume') - # remove source LV copy - try: - self._log.append('removing the DM device copy') - self._lvm.dm_remove(self._copy_dm) - except Exception as err: - self._log.append('failed to remove the DM device copy') - else: - self._log.append('successfully removed DM device copy') - - def _prepare(self): - ''' - ''' - self._lvm.reload() - # get source LV handle - try: - lv = self._lvm.get_lv(self._source_vgname, self._source_lvname) - except: - msg = 'failed to fetch LV `%s/%s`' % (self._source_vgname, - self._source_lvname) - self._log.append(msg) - raise DrbdCopyJobError(msg) - # get source LV infos - self._source_path = lv.path - self._source_size = lv.size - self._source_dm = lv.dm_name() - self._source_table = lv.dm_table() - self._drbd_table = None - # we got here, so all went fine - # just a safety - return lv.size > 0 - - def _job(self): - ''' - ''' - # checkpoint - if self._cancelled: - return - - self._lvm.reload() - - with self._mutex_step: - # create drbd meta - try: - self._log.append('creating a drbd meta volume') - # now let's create a meta device for DRBD in the same VG - # MS = C/32768 + 4MiB && MS >= 128MiB - # http://www.drbd.org/users-guide/ch-internals.html - self._meta_lvname = '%s.drbdmeta' % self._source_lvname - self._meta_path = '/dev/%s/%s' % (self._source_vgname, - self._meta_lvname) - self._meta_size = int(max(self._source_size/32768 + 4 * 2**20, - 128 * 2**20)) - self._lvm.get_vg(self._source_vgname).create(self._meta_lvname, - self._meta_size) - self._lvm.reload() - self._meta_dm = self._lvm.get_lv(self._source_vgname, - self._meta_lvname).dm_name() - except Exception as err: - self._log.append('failed to create a drbd meta volume') - raise err - else: - self._log.append('successfully created the drbd meta volume' - ' `%s`' % self._meta_path) - - # checkpoint - if self._cancelled: - return - - with self._mutex_step: - # create a copy of the source LV - try: - self._log.append('creating a device copy of `%s`' % - self._source_path) - self._copy_dm = '%s.%s.copy' % (self._source_vgname, - self._source_lvname) - self._copy_path = '/dev/mapper/%s' % self._copy_dm - self._lvm.dm_create(self._copy_dm, self._source_table) - except Exception as err: - self._log.append('failed to create a device copy') - raise err - else: - self._log.append('successfully copied device as `%s`' % - self._copy_path) - - # checkpoint - if self._cancelled: - return - - with self._mutex_step: - # setup the drbd device - try: - self._log.append('configuring the drbd device `%s`' % - self._drbd.get_path()) - self._drbd.setup(self._copy_path, self._meta_path) - self._drbd_table = '0 %d linear %s 0' % (self._source_size / 512 - , self._drbd.get_path()) - except Exception as err: - self._log.append('failed to configure the drbd device') - raise err - else: - self._log.append('successfully configured the drbd device') - - # blocking checkpoint - # wait for call to connect() - if self._cancelled: - return - self._event_connected_ready.set() - self._event_connected.wait() - if self._cancelled: - return - - with self._mutex_step: - # wait for a connection with the remote host - try: - self._log.append('waiting for node connection') - self._drbd.wait_connection() - except Exception as err: - self._log.append('failed to establish a connection') - raise err - else: - self._log.append('connected with a remote node') - - # blocking checkpoint - # wait for call to role() - if self._cancelled: - return - self._event_rolechosen_ready.set() - self._event_rolechosen.wait() - if self._cancelled: - return - - # wait for the sync completion - try: - self._log.append('waiting for initial device sync') - self._drbd.wait_sync() - except Exception as err: - import traceback - traceback.print_exc() - self._log.append('failed to sync the DRBD') - raise err - else: - self._log.append('DRBD is sync\'ed !') - - # sleep until the DRBD is shutdown - self._log.append('job entering sleep') - self._event_sleep_ready.set() - self._event_sleep.wait() - self._log.append('job leaving sleep') - - def drbd_get_port(self): - ''' - ''' - return self._drbd.get_port() - - def drbd_connect(self, remote_addr, remote_port): - ''' - ''' - if not self._event_connected.is_set(): - if not self._event_connected_ready.is_set(): - self._log.append('can\'t connect right now, waiting for ' - 'initialization to complete...') - self._event_connected_ready.wait() - with self._mutex_step: - # set endpoint - self._drbd.connect(remote_addr, remote_port) - # unlock any connection wait - self._event_connected.set() - else: - raise DrbdCopyJobError('already connected') - - def drbd_role(self, primary=None): - ''' - ''' - if self._event_connected.is_set(): - self._event_rolechosen_ready.wait() - if primary is True or primary is False: - # change node role - if primary: - self._log.append('switching to primary mode') - self._drbd.primary() - else: - self._log.append('switching to secondary mode') - self._drbd.secondary() - # unlock any role change wait - self._event_rolechosen.set() - else: - raise DrbdCopyJobError('cannot change role, not connected yet') - - def drbd_waitsync(self): - ''' - ''' - self._log.append('waiting for sync') - # wait for initial sync and setup to be completed - self._event_sleep_ready.wait() - # wait for additionnal DRBD sync - with self._mutex_step: - self._drbd.wait_sync() - - def drbd_takeover(self, state): - ''' - ''' - # FIXME comment that - # FIXME check events - with self._mutex_step: - # - if state is True: - if self._drbd_table is not None: - table = self._drbd_table - else: - raise DrbdCopyJobError('no DRBD table available yet') - else: - table = self._source_table - # - self._lvm.dm_set_table(self._source_dm, table) - - def drbd_status(self): - ''' - ''' - return self._drbd.status() - - -class TCPTunnelJob(BaseJob): - ''' - ''' - # tcp port range used for tunnels - TCP_PORT_MIN = 44000 - TCP_PORT_MAX = 45999 - # global port list - _ports_used = [] - _ports_used_mutex = RWLock() - - def __init__(self, manager): - ''' - ''' - super(TCPTunnelJob, self).__init__(manager) - self._mutex = Lock() - self._event_client_conf = Event() - self._event_server_conf = Event() - self._event_server_bound = Event() - self._server_port = None - # set here because of _cancel() - self._sock_client = None - self._sock_server_listen = None - self._sock_server = None - - def _cancel(self): - ''' - ''' - # explicitely close connections, to release blocking socket calls - if self._sock_client: - try: - self._sock_client.close() - except: pass - if self._sock_server_listen: - try: - self._sock_server_listen.close() - except: pass - if self._sock_server: - try: - self._sock_server.close() - except: pass - - def _cleanup(self): - ''' - ''' - # unlock events - self._event_client_conf.set() - self._event_server_conf.set() - self._event_server_bound.set() - # destroy sockets - self._sock_client = None - self._sock_server_listen = None - self._sock_server = None - - def _validate(self): - ''' - ''' - return True # FIXME maybe - - def _prepare(self): - ''' - ''' - return True # FIXME maybe - - def _job(self): - ''' - ''' - try: - # wait for tunnel configuration for server - if not self._cancelled: - self._log.append('waiting for tunnel server configuration') - self._event_server_conf.wait() - - ## setup server - # create listening socket - self._sock_server_listen = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # bind listening socket to a port - self._server_port = None - for port in range(self.TCP_PORT_MIN, self.TCP_PORT_MAX): - # find a port to bind to - try: - with self._ports_used_mutex.write: - if port not in self._ports_used: - addr = '127.0.0.1' if self._server_local else '0.0.0.0' - self._sock_server_listen.bind((addr, port)) #FIXME - self._server_port = port - self._ports_used.append(port) - break - except socket.error: - pass - if not self._server_port: - msg = 'no port to listen to' - self._log.append(msg) - raise TCPTunnelJobError(msg) - else: - self._event_server_bound.set() - self._log.append('server bound to `%s:%d`' % (addr, - self._server_port)) - - # wait for tunnel configuration for client - if not self._cancelled: - self._log.append('waiting for tunnel client configuration') - self._event_client_conf.wait() - - # loop and reconnect forever untill cancellation - while not self._cancelled: - - ## wait for a client to connect - self._log.append('waiting for a client to connect') - self._sock_server_listen.settimeout(1) - self._sock_server_listen.listen(1) - self._sock_server = None - while not self._cancelled and self._sock_server is None: - try: - self._sock_server, local_addr = self._sock_server_listen.accept() - self._log.append('client connected from `%s`' % (local_addr,)) - except socket.timeout: - pass - if self._sock_server is None: - break # job cancelled - - ## connect to the endpoint - timeout = 15 # FIXME must be argument - retry_interval = 5 - # FIXME detection of socket type - if isinstance(self._endpoint, str): - self._sock_client = socket.socket(socket.AF_UNIX, - socket.SOCK_STREAM) - else: - self._sock_client = socket.socket(socket.AF_INET, - socket.SOCK_STREAM) - self._sock_client.settimeout(timeout) - connected = False - while not self._cancelled and not connected: - try: - self._log.append('connecting to `%s`' % (self._endpoint,)) - self._sock_client.connect(self._endpoint) - connected = True - except socket.timeout: - self._log.append('no response after %d seconds' % timeout) - except socket.error: - self._log.append('socket error (connection refused ?)') - time.sleep(retry_interval) - if self._cancelled: - break - if not connected: - continue - - ## tunnel data between sockets - # init socket poller - mask_ro = select.EPOLLIN ^ select.EPOLLERR ^ select.EPOLLHUP - mask_rw = mask_ro ^ select.EPOLLOUT - poller = select.epoll() - poller.register(self._sock_server, mask_ro) - poller.register(self._sock_client, mask_ro) - # forward data until a connection is closed - buf_server = '' - buf_client = '' - empty_buf = 0 - connected = True - while not self._cancelled and connected: - # wait for events on both sockets - poll = poller.poll() - for fd, events in poll: - # events are for server socket - if fd == self._sock_server.fileno(): - # read available - if events & select.EPOLLIN: - # read incoming data - try: - read = self._sock_server.recv(4096) - except socket.error: - connected = False - if not len(read): - empty_buf += 1 - else: - empty_buf = 0 - buf_server += read - # set the other socket to notify us when it's - # available for writing - if len(buf_server): - poller.modify(self._sock_client, mask_rw) - # write available - if events & select.EPOLLOUT: - # try to send the whole buffer - try: - sent = self._sock_server.send(buf_client) - except socket.error: - connected = False - # drop sent data from the buffer - buf_client = buf_client[sent:] - # if the buffer becomes empty, stop write polling - if not len(buf_client): - poller.modify(self._sock_server, mask_ro) - # events for client socket - else: - # read available - if events & select.EPOLLIN: - # read incoming data - try: - read = self._sock_client.recv(4096) - except socket.error: - connected = False - if not len(read): - empty_buf += 1 - else: - empty_buf = 0 - buf_client += read - # set the other socket to notify us when it's - # available for writing - if len(buf_client): - poller.modify(self._sock_server, mask_rw) - # write available - if events & select.EPOLLOUT: - # try to send the whole buffer - try: - sent = self._sock_client.send(buf_server) - except socket.error: - connected = False - # drop sent data from the buffer - buf_server = buf_server[sent:] - # if the buffer becomes empty, stop write polling - if not len(buf_server): - poller.modify(self._sock_client, mask_ro) - if empty_buf >= 10: - connected = False - if connected is False: - self._log.append('disconnected') - try: - self._sock_server.close() - except: pass - try: - self._sock_client.close() - except: pass - except Exception as err: - traceback.print_exc() - raise err - - def tunnel_get_server_port(self): - ''' - ''' - self._event_server_bound.wait() - return self._server_port - - def tunnel_connect(self, endpoint): - ''' - ''' - with self._mutex: - if not self._event_client_conf.is_set(): - self._endpoint = endpoint - self._event_client_conf.set() - else: - raise TCPTunnelJobError('remote endpoint already set') - - def tunnel_listen(self, local): - ''' - ''' - with self._mutex: - if not self._event_server_conf.is_set(): - self._server_local = local is True - self._event_server_conf.set() - else: - raise TCPTunnelJobError('server parameters already set') +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +from threading import Thread +from .logging import Logger + + +class JobManager(Thread, object): + + def __init__(self, logger): + + self._logger = logger diff --git a/ccnode/kernel.py b/ccnode/kernel.py new file mode 100644 index 0000000..7bf4547 --- /dev/null +++ b/ccnode/kernel.py @@ -0,0 +1,655 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +from threading import Lock, Event, Thread +from time import sleep +from sjrpc.client import SimpleRpcClient +from sjrpc.utils import ConnectionProxy, pure +from sjrpc.core import RpcError +from .logging import Logger +from .jobs import JobManager +from .threading import RWLock +from . import modules + +pure = pure + + +class CoreError(Exception): + + pass + + +class Core(object): + + def __init__(self, logger, configuration): + + self._logger = logger + self._config = configuration + self._modmgr = ModuleManager(self._logger, self._config) + + # big lock for events synchronization + self._mutex = Lock() + # state initialized as shutdown + self._shutdown = Event() + self._shutdown.set() + + def run(self): + + # never run twice (should not happen, but let's be sure) + with self._mutex: + if self._shutdown.is_set(): + self._shutdown.clear() + else: + raise CoreError('already running') + + # exceptions in this block will force a shutdown + try: + ### create rpc client + rpc_handler = Handler(self._logger, "core") + rpc_manager = SimpleRpcClient.from_addr( + self._config.get('address'), + self._config.get('port'), + enable_ssl = self._config.get('ssl'), + default_handler = rpc_handler) + rpc_proxy = ConnectionProxy(rpc_manager) + # FIXME bad API workaround + self._rpc_manager = rpc_manager + + # FIXME bad API, move that to __init__ and clean handler each time + # we re-enter run() + self._modmgr.set_core_handler(rpc_handler) + + ### run the sjrpc main loop in a separate thread + ### because we need it for authentication + rpc_thread = Thread(name='sjrpc', target=rpc_manager.run) + rpc_thread.start() + + ### authenticate with the server and set node role + while rpc_thread.is_alive(): + try: + self._logger.debug(1, 'sending authentication request') + # send credential and fetch the node role + role = rpc_proxy.authentify(self._config.get('login'), + self._config.get('password')) + # authentication failed + except RpcError as err: + # bad credentials + if err.exception == 'AuthenticationError': + self._logger.info('authentication failed,' + ' retrying soon...') + # other errors + else: + self._logger.warning('unknown error during' + ' authentication: %s', err) + # wait before trying again + sleep(10) + + # authentication succeeded + else: + self._logger.info('authentication with server succeeded') + + # received hypervisor role: we obey local settings + if role == 'hv': + self._logger.info('server affected hypervisor role') + if not self._config.get('virtualization'): + self._logger.warning('virtualization features are' + ' locally disabled by configuration, cannot' + ' comply with server-affected role') + + # received host role: disabling virtualization + elif role == 'host': + self._logger.info('server affected node role,' + ' disabling virtualization features') + self._config.update('virtualization', False) + + # unknown role, core should be restarted to close connection + else: + self._logger.error('wrong role affected by' + ' server: %s', role) + raise CoreError('wrong role affected') + # quit auth loop + break + + ### check a last time that sjrpc is running + if not rpc_thread.is_alive(): + self._logger.critical('sjrpc main loop terminated too soon') + raise CoreError('lost sjrpc main loop') + + ### load every module usable on this system + try: + self._modmgr.load() + except ModuleManagerError as err: + self._logger.error('initial module load failure: %s', err) + + ### wait for the sjrpc main loop to die + while True: + rpc_thread.join(3600) + if not rpc_thread.is_alive(): + self._logger.info('''sjrpc returned from it's main loop''') + break + else: + self._logger.info('*** hourly life report ***') + + except: + raise + + finally: + self.shutdown() + + def shutdown(self): + + with self._mutex: + if not self._shutdown.is_set(): + # unload every module + # TODO currently will wait forever if a module blocks + self._logger.warning('unloading all modules') + self._modmgr.unload() + # unlock main loop + self._logger.warning('killing main loop') + # kill sjrpc main loop + # FIXME bad API + self._rpc_manager.shutdown() + self._shutdown.set() + + +class ModuleManagerError(Exception): + + pass + + +class ModuleManager(object): + + def __init__(self, logger, config): + + self._logger = logger + self._config = config + self._load_mutex = Lock() + self._dict_mutex = RWLock() + # maps a module name to the python module of this cc-node module :-) + # so it usually contains all available modules + self._mod_loaded = {} + # maps module names to module handles (instance of initialized module) + # so these are currently 'running' modules + self._mod_handles = {} + + # Note: the only methods (load and unload) doing writes on _mod_* dicts + # will lock _load_mutex, so for the sake of simplicity when we're + # in a block protected by this mutex, we'll only use + # _dict_mutex.write as _dict_mutex.read becomes useless there + + def _reload_list(self): + + self._logger.debug(1, 'reloading modules package') + # FIXME maybe use _reload_module and benefit from error checks + reload(modules) + + if not isinstance(modules.LIST, list): + self._logger.debug(1, 'no module list in module package') + modules.LIST = [] + + def _reload_module(self, name): + + self._logger.debug(1, 'reloading module code: %s', name) + with self._dict_mutex.write: + try: + # FIXME is there a better way to handle import VS reload ? + # FIXME do absolute import, but preserving package namespace + # so we can access module with 'modules.foobar' + # FIXME code injection (don't use exec or sanitize input) + if name not in self._mod_loaded: + exec('import ccnode.modules.%s' % name) + else: + exec('reload(modules.%s)' % name) + + except ImportError: + raise ModuleManagerError('%s: import error, the module or a' + ''' dependancy was not found''' % name) + + except KeyboardInterrupt: + raise + + except Exception: + self._logger.backtrace() + raise ModuleManagerError('%s: error in module code, backtrace' + ' was sent to debug logging' % name) + else: + self._mod_loaded[name] = getattr(modules, name) + + def _mod_validate(self, name, module): + + if not hasattr(module, 'Module'): + self._logger.error('module with no definition: %s', name) + return False + + elif not issubclass(module.Module, BaseModule): + self._logger.error('module does not inherint BaseModule: %s', name) + return False + + elif not (hasattr(module.Module, 'mod_dependancies') and + callable(module.Module.mod_dependancies)): + # TODO check the prototype too + self._logger.error('module is missing the mod_dependancies' + ' method: %s', name) + return False + + elif not (hasattr(module.Module, 'mod_initialize') and + callable(module.Module.mod_initialize)): + # TODO check the prototype too + self._logger.error('module is missing the mod_initialize' + ' method: %s', name) + return False + + return True + + def load(self, names=None): + + # read the note about locking in __init__() + + with self._load_mutex: + # reload module index + self._reload_list() + + # if no name is provided, we'll load all modules + if isinstance(names, list): + to_load = names + elif isinstance(names, str): + to_load = [names] + else: + to_load = modules.LIST + + for mod_name in to_load: + ### skip name if already loaded or not in package + if mod_name in self._mod_handles: + self._logger.debug(1, 'skipping module already' + ' initialized: %s', mod_name) + # abort loading this module + continue + elif mod_name not in modules.LIST: + self._logger.debug(1, 'not loading unindexed' + ' module: %s', mod_name) + # abort loading this module + continue + + ### (re)load module code + try: + self._reload_module(mod_name) + with self._dict_mutex.write: + mod = self._mod_loaded[mod_name] + if not self._mod_validate(mod_name, mod): + raise ModuleManagerError('validation failed') + except ModuleManagerError as err: + self._logger.error('failed to load module: %s', err) + # abort loading this module + continue + + ### (re)load dependancies and build deptree + deptree = _DepTree() + # add our current module and it's deps + deptree.add(mod_name, mod.Module.mod_dependancies()) + # deptree tells us what to load + load_error = False + while not load_error: + # get one name to load + dep_name = deptree.next_missing_dep() + # exit if over + if not dep_name: + break + # (re)load it, _only if not in use_ + try: + if not dep_name in self._mod_handles: + self._reload_module(dep_name) + with self._dict_mutex.write: + dep = self._mod_loaded[dep_name] + if not self._mod_validate(dep_name, dep): + raise ModuleManagerError('validation failed') + except ModuleManagerError as err: + self._logger.error('failed to load dependancy' + ' module: %s', err) + # stop the dep scan when load failed + load_error = True + continue + # add the deps of this new mod + deptree.add(dep_name, dep.Module.mod_dependancies()) + # abort all in case we couldn't load a dep + if load_error: + continue + + ### now that all modules are loaded, we should initialize + ### them in the right order, which is provided by _DepTree + try: + for name in deptree.ordered_init_list(): + # ONLY if not initialized yet + if name not in self._mod_handles: + self._logger.debug(1, 'initializing module: %s', + name) + mod = self._mod_loaded[name] + # init the module + if not mod.Module.mod_initialize(self): + # module doesn't wish to load, abort + # (yes it can, and that's not an error) + self._logger.info('module refused to load: %s', + name) + break + # add to list of initialized mods if we have an obj + if (hasattr(mod.Module, 'handle') and + isinstance(mod.Module.handle, mod.Module)): + self._mod_handles[name] = mod.Module.handle + self._logger.info('module initialized: %s', + name) + else: + self._logger.error('module did not provide a' + ' valid instance handle: %s', name) + break + # dependancy error means we couldn't init anything + except _DepTreeError as err: + self._logger.error('a dependancy error prevented loading' + ' of module %s: %s', mod_name, err) + # abort all, no module was initialized + continue + # if any error occur during initialization, can't continue + # with a missing module. we leave things as they are. + except ModuleError as err: + self._logger.error('failed to initialize module %s: %s', + name, err) + # abort all, hopefully the module cleaned it's ressources + continue + except KeyboardInterrupt: + raise + except Exception as err: + self._logger.error('miserable failure during module' + ' initialization of `%s` go check the code' + ' immediately, backtrace in debug log: %s', name, err) + self._logger.backtrace() + # abort all, hopefully the module cleaned it's ressources + continue + + # raise error if the job is not complete + # FIXME should not reporte error when mod_initialize() was False + missing = set(to_load) - set(self._mod_handles.keys()) + if len(missing): + raise ModuleManagerError('could not load: %s' % + ', '.join(missing)) + + def unload(self, names=None): + + # please read the note about locking in __init__() + + with self._load_mutex: + if isinstance(names, list): + to_unload = names + elif isinstance(names, str): + to_unload = [names] + else: + to_unload = self._mod_loaded.keys() + + # FIXME + # check tah _loaded do not contains modules with deps to to_unload + + # FIXME should disappear, see Core.run() + def set_core_handler(self, handler): + + self._core_handler = handler + + def get_handler(self): + + return self._core_handler + + def get_logger(self): + + return self._logger + + def get_configuration(self): + + return self._config + + def get(self, name): + + with self._dict_mutex.read: + if name in self._mod_handles: + return self._mod_handles[name] + else: + raise ModuleManagerError('access to uninitialized module: %s' + % name) + + +class _DepTreeError(Exception): + + pass + + +class _DepTree(object): + + def __init__(self): + + self._mod_deps = {} + + def add(self, name, deps): + + # save it in our mod->deps mapping + self._mod_deps[name] = deps + + def next_missing_dep(self): + + # return the first module present in a dependancies list + # but not registered in the mod->deps mapping + for mod, deps in self._mod_deps.iteritems(): + for dep in deps: + if dep not in self._mod_deps: + return dep + + # or None when we do not miss anything anymore + return None + + def ordered_init_list(self): + + # + # How it works: + # + # * first I create a directed graph of all modules linked + # by dependancies, despite the class name it's not always + # a tree + # * then I ensure that we have no cycle in it, or it means + # unresolvable deps and the process stops. actually this is + # done during the following step + # * finally the list is built starting with modules that have + # no child, level-by-level up to the last parent(s) + # + # Graph node: + # (mod_name, {deps name:@ref}) + # + + ### create the directed graph of dependancies + graph = [] + for mod_name, mod_deps_list in self._mod_deps.iteritems(): + # create node tuple + mod_deps = dict([(name, None) for name in mod_deps_list]) + mod = (mod_name, mod_deps) + # iter on nodes to make arcs + for node in graph: + n_name, n_deps = node + # add node->mod arc + if mod_name in n_deps: + n_deps[mod_name] = mod + # add mod->node arc + for dep_name, dep in mod_deps.iteritems(): + if n_name == dep_name: + dep = node + # add the new node to the graph + graph.append(mod) + + ### build an ideal module initialization list that obey dependancies + # scan for the initial level: modules with no dependancies + level = [] + for node in graph: + n_name, n_deps = node + if not len(n_deps): + level.append(node) + + # stop if we've lost already + if not len(level) and len(self._mod_deps): + raise _DepTreeError('cross-dependancies on all the modules') + + # now iter on next levels + load_list = [] + while True: + # extend the list with current level + for n_name, n_deps in level: + load_list.append(n_name) + # build the next level + next_level = [] + for node in graph: + n_name, n_deps = node + # only add _new_ nodes that are fulfiled with the current list + if (n_name not in load_list + and set(n_deps.keys()) <= set(load_list)): + next_level.append(node) + # stop when the next level is empty + if not len(next_level): + # if this is a cross-deps situation, abort + if len(load_list) < len(graph): + # FIXME enhance the message explaining what modules remain + # to load + raise _DepTreeError('cross-dependancies loop') + # we're finished + else: + break + # good to go to the next level + else: + level = next_level + + return load_list + + +class ModuleError(Exception): + + pass + + +class BaseModule(object): + + _DEPS = [] + + @classmethod + def mod_dependancies(cls): + + return cls._DEPS if isinstance(cls._DEPS, list) else [] + + @classmethod + def mod_initialize(cls, manager): + + if not hasattr(cls, 'loadable') or (callable(cls.loadable) + and cls.loadable()): + cls.handle = cls(manager) + return True + else: + return False + + @classmethod + def mod_destroy(cls): + + if hasattr(self, 'destroy') and callable(self.destroy): + self.destroy() + del cls.handle + + def __init__(self, manager): + + if hasattr(self, 'init') and callable(self.init): + self.init(manager) + + +class HandlerError(Exception): + + pass + + +class Handler(object): + + def __init__(self, logger, name): + + self._logger = logger + self._name = self.__class__.__name__ if name is None else name + self._mutex = RWLock() + self._procedures = {} + self._children = [] + + def __getitem__(self, name): + + if name.startswith('_'): + raise AttributeError('attribute `%s` is private' % name) + + search = self._search_procedure(name) + + if search is not None: + handler, procedure = search + self._logger.debug(1, 'handler access for: %s->%s', handler, name) + return procedure + else: + self._logger.info('procedure `%s` is not registered in subtree' + ' of handler `%s`' % (name, self._name)) + raise KeyError('procedure `%s` is not registered' % name) + + def _search_procedure(self, name): + + with self._mutex.read: + # return local procedure if present, with handler name + if name in self._procedures: + return (self._name, self._procedures[name]) + + # search in sub-handlers + for handler in self._children: + search = handler._search_procedure(name) + if search is not None: + return search + + # not found at all + return None + + def register(self, handler): + + with self._mutex.write: + if handler not in self._children: + self._children.append(handler) + else: + raise HandlerError('handler `%s` is already registered in `%s`' + % (handler._name, self._name)) + + def unregister(self, handler): + + with self._mutex.write: + while handler in self._children: + self._children.remove(handler) + + def procedure_register(self, name, procedure): + + with self._mutex.write: + if name not in self._procedures: + self._procedures[name] = procedure + else: + raise HandlerError('procedure `%s` is already registered' + ' in `%s`' % (name, self._name)) + + def procedure_unregister(self, name): + + with self._mutex.write: + if name in self._procedures: + self._procedures.pop(name) diff --git a/ccnode/kvm.py b/ccnode/kvm.py deleted file mode 100644 index 9d42035..0000000 --- a/ccnode/kvm.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -from libvirtwrapper import LibvirtHypervisor - - -class KvmHypervisor(LibvirtHypervisor): - ''' - Base class of a Kvm Hypervisor - ''' - _instance = None - - def __init__(self): - ''' - ''' - super(KvmHypervisor, self).__init__('kvm') - - def __new__(cls, *args, **kwargs): - ''' - .. note:: - We use singleton design pattern to force only a single instance - of our libvirt hypervisor handle, it's essential since we connect - with libvirt only on localhost so we must assure one single - connection to the hypervisor - ''' - if cls._instance is None: - cls._instance = super(KvmHypervisor, cls).__new__(cls, *args, - **kwargs) - return cls._instance diff --git a/ccnode/launcher.py b/ccnode/launcher.py new file mode 100644 index 0000000..ea45eef --- /dev/null +++ b/ccnode/launcher.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +from signal import signal, SIGHUP, SIGUSR1 +from time import sleep +from .logging import Logger, DEBUG, INFO, ERROR +from .config import Configuration +from .kernel import Core +from . import __product__, __version__ + + +class Launcher(object): + # pylint: disable=R0903 + + def __init__(self): + + # initialize core components + self._logger = Logger() + self._logger.info('initializing %s version %s' % (__product__, + str(__version__))) + self._config = Configuration(self._logger) + self._core = Core(self._logger, self._config) + + def _on_sighup(self, signum, frame): + # pylint: disable=W0613 + + self._logger.warning('SIGHUP received, shutting down core') + self._core.shutdown() + + def _on_sigusr1(self, signum, frame): + # pylint: disable=W0613 + + self._logger.info('SIGUSR1 received, reloading configuration file and' + ' logging settings') + self._reload_conf() + + def _reload_conf(self): + + # (re)load configuration file + self._config.parse_file() + + # (re)apply logging configuration + # verbosity level + verbosity = self._config.get('verbosity') + if verbosity >= 1: + self._logger.set_level(DEBUG) + self._logger.set_debug_level(verbosity) + else: + self._logger.set_level(INFO) + # stderr/syslog output selection + std_output = self._config.get('stdout') + self._logger.set_stderr_enabled(std_output) + self._logger.set_syslog_enabled(not std_output) + + def main(self): + + # read command line arguments + self._config.parse_arguments() + + # register hangup signal to restart core + signal(SIGHUP, self._on_sighup) + + # register USR1 signal to reload configuration file + # and apply logging settings + signal(SIGUSR1, self._on_sigusr1) + + # run the core untill permanent shutdown is requested + while True: + try: + # (re)load configuration file and apply logging settings + self._reload_conf() + + # start core main loop + self._core.run() + sleep(0.5) + + except KeyboardInterrupt: + self._logger.warning('SIGINT received, shutting down core' + ' post-mortem') + self._core.shutdown() + break + + except Exception: + self._logger.error('unexpected error raised by core') + self._logger.backtrace(priority=ERROR) + sleep(5) diff --git a/ccnode/libvirtwrapper.py b/ccnode/libvirtwrapper.py deleted file mode 100644 index a96adfe..0000000 --- a/ccnode/libvirtwrapper.py +++ /dev/null @@ -1,831 +0,0 @@ -# -*- coding: utf-8 -*- - -import libvirt -import psutil -import xml.dom.minidom -import os -from logging import error, warning, info, debug -from time import sleep -from common import Hypervisor, VM, Storage, StoragePool, StorageVolume -from utils import RWLock -from errors import (HypervisorError, VMError, StorageError, StoragePoolError, - StorageVolumeError) - - -KVM_LIBVIRT_SESSION = 'qemu:///system' -XEN_LIBVIRT_SESSION = 'xen:///' - -KILOBYTE_DIV = 1024 -MEGABYTE_DIV = KILOBYTE_DIV * 1024 -GIGABYTE_DIV = MEGABYTE_DIV * 1024 - - -#### hypervisor - -class LibvirtHypervisor(Hypervisor): - ''' - ''' - - def __init__(self, hv_type): - ''' - ''' - super(LibvirtHypervisor, self).__init__() - try: - if hv_type == 'kvm': - warning("LibvirtHypervisor: initialized as KVM") - self._lvcon_handle = libvirt.open(KVM_LIBVIRT_SESSION) - elif hv_type == 'xen': - warning("LibvirtHypervisor: initialized as Xen") - self._lvcon_handle = libvirt.open(XEN_LIBVIRT_SESSION) - else: - raise NotImplemented('Unknown hypervisor type') - except libvirt.libvirtError as error: - raise HypervisorError('libvirt cannot connect to hypervisor') - - self._hv_type = hv_type - self._sto_handle = LibvirtStorage(self) - self._vm_cache_running = {} - self._vm_cache_defined = {} - self._vm_cache = {} - self._vm_cache_lock = RWLock() - - def scheduler_run(self): - self._cache_vm_rebuild() - - def _cache_vm_rebuild(self): - ''' - ''' - running = {} - defined = {} - - try: - for dom_id in self._lvcon_handle.listDomainsID(): - vm = LibvirtVm(self, self._lvcon_handle.lookupByID(dom_id)) - running[vm.get_name()] = vm - for dom_name in self._lvcon_handle.listDefinedDomains(): - vm = LibvirtVm(self, self._lvcon_handle.lookupByName(dom_name)) - defined[vm.get_name()] = vm - except Exception as err: - debug("_cache_vm_rebuild: abort, caught exception `%r`:`%s`", err, - err) - else: - with self._vm_cache_lock.write: - #debug("_cache_vm_rebuild: running: %s" % running) - #debug("_cache_vm_rebuild: defined: %s" % defined) - #debug("_cache_vm_rebuild: old-running: %s" % self._vm_cache_running) - #debug("_cache_vm_rebuild: old-defined: %s" % self._vm_cache_defined) - self._vm_cache_running = running - self._vm_cache_defined = defined - self._vm_cache = self._vm_cache_running - self._vm_cache.update(self._vm_cache_defined) - - def get_hv_type(self): - ''' - ''' - return self._hv_type - - def get_hv_version(self): - ''' - ''' - version = None - try: - data = self._lvcon_handle.getVersion() - if data: - version = data - except: - pass - return version - - def get_libvirt_version(self): - ''' - ''' - version = None - try: - data = self._lvcon_handle.getLibVersion() - if data: - version = data - except: - pass - return version - - def get_cpu_threads(self): - ''' - ''' - return self._lvcon_handle.getInfo()[7] * self.get_cpu() - - def get_cpu_frequency(self): - ''' - ''' - return self._lvcon_handle.getInfo()[3] - - def storage(self): - ''' - ''' - return self._sto_handle - - def vm_define(self, data): - ''' - ''' - vm = self._lvcon_handle.defineXML(data) - self._cache_vm_rebuild() - if hasattr(vm, 'name'): - return vm.name() - else: - raise HypervisorError('VM not defined properly') - - def vm_list(self): - ''' - ''' - with self._vm_cache_lock.read: - return self._vm_cache.keys() - - def vm_list_running(self): - ''' - ''' - running = [] - with self._vm_cache_lock.read: - for vm_name, vm in self._vm_cache_running.iteritems(): - if vm.is_active(): - running.append(vm_name) - return running - - def vm_list_stopped(self): - ''' - ''' - with self._vm_cache_lock.read: - return self._vm_cache_defined.keys() - - def vm_list_paused(self): - ''' - ''' - paused = [] - with self._vm_cache_lock.read: - for vm_name, vm in self._vm_cache_running.iteritems(): - if vm.is_paused(): - paused.append(vm_name) - return paused - - def vm_get(self, name): - ''' - ''' - if name in self.vm_list(): - try: - with self._vm_cache_lock.read: - return self._vm_cache[name] - except: - raise HypervisorError('VM `%s` has vanished' % name) - else: - raise HypervisorError('host has no VM named `%s`' % name) - - -#### storage - -class LibvirtStorage(Storage): - ''' - ''' - - def __init__(self, hypervisor): - ''' - ''' - if isinstance(hypervisor, LibvirtHypervisor): - self._hv_handle = hypervisor - else: - raise TypeError('Expected `%s` given `%s`' % (LibvirtHypervisor, - hypervisor)) - - self._pool_cache_running = {} - self._pool_cache_defined = {} - self._pool_cache = {} - self._pool_cache_lock = RWLock() - - def _pool_cache_rebuild(self): - ''' - ''' - with self._pool_cache_lock.write: - self._pool_cache_running = {} - self._pool_cache_defined = {} - self._pool_cache = {} - - for name in self._hv_handle._lvcon_handle.listStoragePools(): - pool = LibvirtStoragePool(self, - self._hv_handle._lvcon_handle.storagePoolLookupByName(name)) - self._pool_cache_running[pool.get_name()] = pool - - for name in self._hv_handle._lvcon_handle.listDefinedStoragePools(): - pool = LibvirtStoragePool(self, - self._hv_handle._lvcon_handle.storagePoolLookupByName(name)) - self._pool_cache_defined[pool.get_name()] = pool - - self._pool_cache = self._pool_cache_running - self._pool_cache.update(self._pool_cache_defined) - - def pool_list(self): - ''' - ''' - if not self._pool_cache: - self._pool_cache_rebuild() - with self._pool_cache_lock.read: - return self._pool_cache.keys() - - def pool_get(self, name): - ''' - ''' - if name in self.pool_list(): - try: - with self._pool_cache_lock.read: - return self._pool_cache[name] - except: - raise StorageError('storage pool `%s` vanished' % name) - else: - raise StorageError('no storage pool with name `%s`' % name) - - def capacity(self): - ''' - ''' - capacity = 0 - for pool_name in self.pool_list(): - pool = self.pool_get(pool_name) - capacity += pool.get_space_capacity() - return capacity - - def find_volumes(self, path=None, name=None): - ''' - ''' - volumes = [] - if path is not None or name is not None: - for pool_name in self.pool_list(): - pool = self.pool_get(pool_name) - for vol_name in pool.volume_list(): - vol = pool.volume_get(vol_name) - if (path is not None and vol.get_path() == path) \ - or (name is not None and vol.get_name() == name): - volumes.append(vol) - return volumes - - -class LibvirtStoragePool(StoragePool): - ''' - ''' - - def __init__(self, storage, libvirt_pool): - ''' - ''' - if isinstance(storage, LibvirtStorage): - self._sto_handle = storage - else: - raise TypeError('Expected `%s` given `%s`' % (LibvirtStorage, - storage)) - if isinstance(libvirt_pool, libvirt.virStoragePool): - self._lvpool_handle = libvirt_pool - else: - raise TypeError('Expected `%s` given `%s`' % (libvirt.virStoragePool - , libvirt_pool)) - - self._vol_cache = {} - self._vol_cache_lock = RWLock() - - def _vol_cache_rebuild(self): - ''' - ''' - with self._vol_cache_lock.write: - self._vol_cache = {} - if self._lvpool_handle.isActive(): - for name in self._lvpool_handle.listVolumes(): - vol = LibvirtStorageVolume(self, - self._lvpool_handle.storageVolLookupByName(name)) - self._vol_cache[vol.get_name()] = vol - - def volume_list(self): - ''' - ''' - if not self._vol_cache: - self._vol_cache_rebuild() - with self._vol_cache_lock.read: - return self._vol_cache.keys() - - def volume_get(self, name): - ''' - ''' - if name in self.volume_list(): - try: - with self._vol_cache_lock.read: - return self._vol_cache[name] - except: - raise StoragePoolError('volume `%s` has vanished from pool `%s`' - %(name, self.get_name())) - else: - raise StoragePoolError('pool `%s` has no volume `%s`' % ( - self.get_name(), name)) - - def volume_create(self, name, size): - ''' - ''' - xml = ''' - - %(name)s - %(capacity)i - - ''' % { - 'name' : name, - 'capacity' : size - } - try: - vol = self._lvpool_handle.createXML(xml, 0) - if isinstance(vol, libvirt.virStorageVol): - self._vol_cache_rebuild() - return vol - else: - raise StoragePoolError('volume creation failed for an unknown reason') - except libvirt.libvirtError as err: - raise StoragePoolError('volume creation failed : `%r` : `%s`' % - (err, err)) - - def get_name(self): - ''' - ''' - name = None - try: - data = self._lvpool_handle.name() - if data: - name = data - except libvirt.libvirtError: - pass - return name - - def get_source_name(self): - ''' - ''' - name = None - try: - xroot = xml.dom.minidom.parseString(self._lvpool_handle.XMLDesc(0)) - xpool = xroot.getElementsByTagName('pool').pop() - xsource = xpool.getElementsByTagName('source').pop() - xname = xpool.getElementsByTagName('name').pop() - name = xname.childNodes[0].nodeValue - except libvirt.libvirtError: - pass - return name - - def get_source_format(self): - ''' - ''' - format = None - try: - xroot = xml.dom.minidom.parseString(self._lvpool_handle.XMLDesc(0)) - xpool = xroot.getElementsByTagName('pool').pop() - xsource = xpool.getElementsByTagName('source').pop() - xformat = xpool.getElementsByTagName('format').pop() - format = xformat.getAttribute('type') - except libvirt.libvirtError: - pass - return format - - def get_type(self): - ''' - ''' - typ = None - try: - xroot = xml.dom.minidom.parseString( - self._lvpool_handle.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)) - xpool = xroot.getElementsByTagName('pool').pop() - typ = xpool.getAttribute('type') - except libvirt.libvirtError: - pass - return typ - - def get_space_capacity(self): - ''' - ''' - try: - return self._lvpool_handle.info()[1] - except libvirt.libvirtError as e: - raise StoragePoolError("can't get pool information (%s)" % e) - - def get_space_free(self): - ''' - ''' - try: - return self._lvpool_handle.info()[3] - except libvirt.libvirtError as e: - raise StoragePoolError("can't get pool information (%s)" % e) - - def get_space_used(self): - ''' - ''' - try: - return self._lvpool_handle.info()[2] - except libvirt.libvirtError as e: - raise StoragePoolError("can't get pool information (%s)" % e) - - -class LibvirtStorageVolume(StorageVolume): - ''' - ''' - - def __init__(self, pool, libvirt_vol): - ''' - ''' - if isinstance(pool, LibvirtStoragePool): - self._pool_handle = pool - else: - raise TypeError('Expected `%s` given `%s`' % (LibvirtStoragePool, - pool)) - if isinstance(libvirt_vol, libvirt.virStorageVol): - self._lvvol_handle = libvirt_vol - else: - raise TypeError('Expected `%s` given `%s`' % (libvirt.virStorageVol, - libvirt_vol)) - - def wipe(self): - ''' - ''' - try: - if self._lvvol_handle.wipe(0): - raise StorageVolumeError('volume wipe failed for an unknown reason') - except libvirt.libvirtError as err: - raise StorageVolumeError('volume wipe failed : `%r` : `%s`' % (err, - err)) - - def delete(self): - ''' - ''' - try: - if self._lvvol_handle.delete(0): - raise StorageVolumeError('volume deletion failed for an unknown reason') - else: - self._pool_handle._vol_cache_rebuild() - except libvirt.libvirtError as err: - raise StorageVolumeError('volume deletion failed : `%r` : `%s`' % - (err, err)) - - def get_pool(self): - ''' - ''' - pool = None - try: - data = self._lvvol_handle.storagePoolLookupByVolume() - if data: - pool = data - except libvirt.libvirtError: - pass - return pool - - def get_name(self): - ''' - ''' - name = None - try: - data = self._lvvol_handle.name() - if data: - name = data - except libvirt.libvirtError: - pass - return name - - def get_space_capacity(self): - ''' - ''' - capacity = None - try: - capacity = self._lvvol_handle.info()[1] - except libvirt.libvirtError: - pass - return capacity - - def get_space_allocation(self): - ''' - ''' - allocated = None - try: - allocated = self._lvvol_handle.info()[2] - except libvirt.libvirtError: - pass - return allocated - - def get_path(self): - ''' - ''' - path = None - try: - path = self._lvvol_handle.path() - except libvirt.libvirtError: - pass - return path - -#### vm - -class LibvirtVm(VM): - - ARCH = { - 'i686' : 'x86', - 'x86_64' : 'x64', - } - - STATUS = ( - 'No state', - 'Running', - 'Blocked on resource', - 'Paused', - 'Shutting down ...', - 'Shutdown', - 'Crashed', - ) - STATUS_STOPPED = [0, 5, 6] - STATUS_RUNNING = [1, 2 , 3, 4] - STATUS_PAUSED = [3] - - def __init__(self, hypervisor, domain): - ''' - ''' - super(LibvirtVm, self).__init__() - if isinstance(domain, libvirt.virDomain): - self._domain = domain - else: - raise TypeError('Need virDomain object given %s' % type(domain)) - - self._hv_handle = hypervisor - self._find_pid() - - def _find_pid(self): - ''' - ''' - result = find_process_id(self.get_uuid()) - if result: - self._pid = int(result.pop()) - else: - self._pid = None - - def hypervisor(self): - ''' - ''' - return self._hv_handle - - def undefine(self): - ''' - ''' - if self._domain.undefine(): - raise VMError('deletion of VM `%s` failed' % self.get_name()) - self._hv_handle._cache_vm_rebuild() - - def migrate(self, libvirt_addr, hv_addr): - ''' - ''' - # special case for xen - if self.hypervisor().get_hv_type() == 'xen': - flags = (libvirt.VIR_MIGRATE_LIVE ^ - libvirt.VIR_MIGRATE_PERSIST_DEST ^ - libvirt.VIR_MIGRATE_UNDEFINE_SOURCE) - lv_uri = 'xen+tcp://%s:%d' % libvirt_addr - mig_uri = 'xenmigr://%s:%d' % hv_addr - # kvm and others - else: - flags = (libvirt.VIR_MIGRATE_LIVE ^ - libvirt.VIR_MIGRATE_PERSIST_DEST ^ - libvirt.VIR_MIGRATE_UNDEFINE_SOURCE ^ - libvirt.VIR_MIGRATE_PEER2PEER ^ - libvirt.VIR_MIGRATE_TUNNELLED) - lv_uri = 'qemu+tcp://%s:%d/system' % libvirt_addr - mig_uri = 'qemu+tcp://%s:%d/system' % hv_addr - - # establish the connection with remote libvirt - rconn = libvirt.open(lv_uri) - - # migrate ! - try: - self._domain.migrate(rconn, flags, None, mig_uri, 0) - except libvirt.libvirtError as err: - # FIXME ignore bogus exception properly - # maybe no more needed since real remote connection is provided - if not ('no domain with matching name' in err.message - or 'Domain not found' in err.message): - raise err - finally: - # close the libvirt connection - rconn.close() - - def power_on(self): - ''' - ''' - try: - self._domain.create() - except libvirt.libvirtError: - raise VMError('`%s` is already running' % self.get_name()) - - def power_off(self): - ''' - ''' - try: - self._domain.destroy() - except libvirt.libvirtError: - raise VMError('`%s` is not running' % self.get_name()) - - def power_shutdown(self): - ''' - ''' - try: - self._domain.shutdown() - except libvirt.libvirtError: - raise VMError('`%s` is not running' % self.get_name()) - - def power_suspend(self): - ''' - ''' - try: - self._domain.suspend() - except libvirt.libvirtError: - raise VMError('`%s` is not running, or already paused' - % self.get_name()) - - def power_resume(self): - ''' - ''' - try: - self._domain.resume() - except libvirt.libvirtError: - raise VMError('`%s` is not paused, or not running' - % self.get_name()) - - def is_active(self): - ''' - ''' - active = None - try: - active = self._domain.info()[0] in self.STATUS_RUNNING - except libvirt.libvirtError: - pass - return active - - def is_paused(self): - ''' - ''' - paused = None - try: - paused = self._domain.info()[0] in self.STATUS_PAUSED - except libvirt.libvirtError: - pass - return paused - - def get_config(self): - ''' - ''' - return self._domain.XMLDesc(0) - - def get_uuid(self): - ''' - ''' - return self._domain.UUIDString() - - def get_name(self): - ''' - ''' - return self._domain.name() - - def get_pid(self): - ''' - ''' - return self._pid - - def get_arch(self): - ''' - ''' - arch = None - try: - # bug #4020 - if self.hypervisor().get_hv_type() == 'xen': - arch = self.hypervisor().get_arch() - else: - xroot = xml.dom.minidom.parseString( - self._domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)) - xdomain = xroot.getElementsByTagName('domain').pop() - xos = xdomain.getElementsByTagName('os').pop() - xtype = xos.getElementsByTagName('type').pop() - xarch = xtype.getAttribute('arch') - if xarch in self.ARCH: - arch = self.ARCH[xarch] - except: - pass - return arch - - def get_cpu_core(self): - ''' - ''' - return self._domain.info()[3] - - def get_cpu_usage(self): - ''' - ''' - usage = None - if self._pid is not None: - try: - p = psutil.Process(self._pid) - sleep(0.2) - usage = p.get_cpu_percent() - except: - pass - return usage - - def get_mem(self): - ''' - ''' - return self._domain.info()[2] * KILOBYTE_DIV - - def get_mem_max(self): - ''' - ''' - return self._domain.info()[1] * KILOBYTE_DIV - - def get_vnc_port(self): - ''' - ''' - port = None - try: - xroot = xml.dom.minidom.parseString( - self._domain.XMLDesc(libvirt.VIR_DOMAIN_XML_SECURE)) - xdomain = xroot.getElementsByTagName('domain').pop() - xgraphics = xdomain.getElementsByTagName('graphics').pop() - data = int(xgraphics.getAttribute('port')) - if data > 0 and data <= 65535: - port = data - except: - pass - return port - - def get_volumes(self): - ''' - ''' - volumes = [] - try: - xroot = xml.dom.minidom.parseString( - self._domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)) - xdomain = xroot.getElementsByTagName('domain').pop() - xdevices = xdomain.getElementsByTagName('devices').pop() - # iter on "disk" devices - for xdisk in xdevices.getElementsByTagName('disk'): - try: - # disks we can handle - if xdisk.getAttribute('device') == 'disk': - # get type - d_type = xdisk.getAttribute('type') - # get backend path - if d_type == 'file': - d_path = xdisk.getElementsByTagName('source').pop()\ - .getAttribute('file') - elif d_type == 'block': - d_path = xdisk.getElementsByTagName('source').pop()\ - .getAttribute('dev') - # FIXME sometimes xen do not report '/dev/' at the - # beginning of block devices, and relative paths - # are non-sense - # Example: vg/myvm instead of /dev/vg/myvm - if not d_path.startswith('/'): - d_path = os.path.join('/dev', d_path) - # search the volume object - if d_type in ['file', 'block']: - volumes.append(self._hv_handle._sto_handle - .find_volumes(path=d_path).pop()) - except Exception as e: - print e - except: - pass - return volumes - - def get_nics(self): - ''' - ''' - nics = [] - try: - xroot = xml.dom.minidom.parseString( - self._domain.XMLDesc(libvirt.VIR_DOMAIN_XML_INACTIVE)) - xdomain = xroot.getElementsByTagName('domain').pop() - xdevices = xdomain.getElementsByTagName('devices').pop() - # iter on "interface" devices - for xint in xdevices.getElementsByTagName('interface'): - nic = {} - try: - # search for network interfaces - if xint.getAttribute('type') in ['bridge']: - # mac - nic['mac'] = xint.getElementsByTagName('mac').pop()\ - .getAttribute('address') - # model - nic['model'] = xint.getElementsByTagName('model').pop()\ - .getAttribute('type') - # source - nic['source'] = xint.getElementsByTagName('source')\ - .pop().getAttribute('bridge') - except: - pass - else: - nics.append(nic) - except: - pass - return nics - - -#### helpers - -def find_process_id(cmd_subchain): - ''' - ''' - return [p.pid for p in psutil.get_process_list() - if p.cmdline.__contains__(cmd_subchain)] diff --git a/ccnode/logging.py b/ccnode/logging.py new file mode 100644 index 0000000..5ace2ef --- /dev/null +++ b/ccnode/logging.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +import syslog +import socket +from sys import stderr +from threading import Lock +from traceback import format_exc +from . import __canonical__, __develmode__ + +DEBUG = syslog.LOG_DEBUG +INFO = syslog.LOG_INFO +WARNING = syslog.LOG_WARNING +ERROR = syslog.LOG_ERR +CRITICAL = syslog.LOG_CRIT + +ALL_PRIO = (DEBUG, INFO, WARNING, ERROR, CRITICAL) +PRIO_TAGS = { + DEBUG : 'D', + INFO : 'I', + WARNING : 'W', + ERROR : 'E', + CRITICAL: 'C', +} + + +class Logger(object): + + def __init__(self): + + self._max_prio = INFO + self._max_debug = 1 + self._mutex = Lock() + self._mcast_sock = None + self._mcast_dst = None + self._syslog_enabled = not __develmode__ + self._stderr_enabled = __develmode__ + syslog.openlog(__canonical__, + syslog.LOG_PID ^ syslog.LOG_NDELAY ^ syslog.LOG_CONS, + syslog.LOG_DAEMON) + + def set_level(self, priority): + + if priority not in ALL_PRIO: + raise ValueError('not an allowed logging priority') + + self._max_prio = priority + + def set_debug_level(self, level): + + if level <= 0: + raise ValueError('debug level is a positive integer') + + self._max_debug = int(level) + + def set_syslog_enabled(self, state): + + self._syslog_enabled = bool(state) + + def set_stderr_enabled(self, state): + + self._stderr_enabled = bool(state) + + def mcast_enable(self, addr, port): + + with self._mutex: + self._mcast_dst = (addr, port) + self._mcast_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, + socket.IPPROTO_UDP) + self._mcast_sock.setsockopt(socket.IPPROTO_IP, + socket.IP_MULTICAST_TTL, 2) + + def mcast_disable(self): + + with self._mutex: + self._mcast_sock = None + self._mcast_dst = None + + def _log(self, priority, level, message, *args): + + if priority not in ALL_PRIO: + priority = CRITICAL + + printable = (priority <= self._max_prio and (level is None + or (priority == DEBUG and level <= self._max_debug))) + + if printable or self._mcast_sock is not None: + try: + if len(args): + message = message % args + + except TypeError: + message = '[PARTIAL] %s' % message + + finally: + if printable: + if self._syslog_enabled: + syslog.syslog(priority, message) + + if self._stderr_enabled: + if isinstance(level, int): + my_lvl = str(level) + else: + my_lvl = ' ' + stderr.write('%s[%s%s]: %s\n' % (__canonical__, + PRIO_TAGS[priority], my_lvl, message)) + + with self._mutex: + if self._mcast_sock is not None: + self._mcast_sock.sendto('%i %s' % + (level, message), self._mcast_dst) + + def debug(self, level, message, *args): + + self._log(DEBUG, level, message, *args) + + def info(self, message, *args): + + self._log(INFO, None, message, *args) + + def warning(self, message, *args): + + self._log(WARNING, None, message, *args) + + def error(self, message, *args): + + self._log(ERROR, None, message, *args) + + def critical(self, message, *args): + + self._log(CRITICAL, None, message, *args) + + def backtrace(self, priority=DEBUG, level=1): + + if priority != DEBUG: + level = None + + for line in format_exc().splitlines(): + self._log(priority, level, 'BT: %s', line) diff --git a/ccnode/lvm.py b/ccnode/lvm.py deleted file mode 100644 index bd76fb6..0000000 --- a/ccnode/lvm.py +++ /dev/null @@ -1,343 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -from errors import LVMError -from utils import Exec, RWLock - -class LVM(object): - ''' - ''' - BIN_PATH = '/sbin' - LVM = os.path.join(BIN_PATH, 'lvm') - LVMDISKSCAN = os.path.join(BIN_PATH, 'lvmdiskscan') - LVDISPLAY = os.path.join(BIN_PATH, 'lvdisplay') - LVCREATE = os.path.join(BIN_PATH, 'lvcreate') - LVCHANGE = os.path.join(BIN_PATH, 'lvchange') - LVCONVERT = os.path.join(BIN_PATH, 'lvconvert') - LVRENAME = os.path.join(BIN_PATH, 'lvrename') - LVEXTEND = os.path.join(BIN_PATH, 'lvextend') - LVREDUCE = os.path.join(BIN_PATH, 'lvreduce') - LVREMOVE = os.path.join(BIN_PATH, 'lvremove') - PVCREATE = os.path.join(BIN_PATH, 'pvcreate') - PVREMOVE = os.path.join(BIN_PATH, 'pvremove') - PVMOVE = os.path.join(BIN_PATH, 'pvmove') - VGCREATE = os.path.join(BIN_PATH, 'vgcreate') - VGCHANGE = os.path.join(BIN_PATH, 'vgchange') - VGEXTEND = os.path.join(BIN_PATH, 'vgextend') - VGREDUCE = os.path.join(BIN_PATH, 'vgreduce') - VGREMOVE = os.path.join(BIN_PATH, 'vgremove') - DMSETUP = os.path.join(BIN_PATH, 'dmsetup') - - # global class lock : do not abuse LVM/DM too much - _mutex = RWLock() - - def __init__(self): - ''' - ''' - self.reload() - - """ - FIXME not working anymore - def __str__(self): - ''' - ''' - s = '' - for vgname, vg in self._vgs.iteritems(): - s += 'VG %s (total: %s, allocated: %s, free: %s)\n' % (vgname, - vg.total, vg.allocated, vg.free) - s += ' * PV(s):\n' - for pv in vg.pvs: - s += (' %s (total: %s, allocated: %s, free: %s, format: %s)' - '\n' % (pv.name, pv.total, pv.allocated, pv.free, pv.format)) - s += ' * LV(s):\n' - for lv in vg.lvs: - s += ' %s (path: %s, size: %s)' % (lv.name, lv.path, - lv.total) - return s - """ - - def _reload_pvs(self): - ''' - ''' - with self._mutex.write: - self._pvs = {} - # execute query command - (rc, output) = Exec.call([self.LVM, - 'pvs', - '--nosuffix', - '--noheadings', - '--units', 'b', - '--separator', '~', - '-o', ('pv_uuid,pv_name,vg_name,pv_fmt,' - 'pv_attr,dev_size,pv_size,pv_free,' - 'pv_used,pv_pe_count,' - 'pv_pe_alloc_count')], - merge_output = False) - if rc != 0: - raise LVMError('error getting list of PV') - # parse command output - for line in output[0].splitlines(): - # extract tokens - ( - p_uuid, - p_name, - p_vg, - p_format, - p_attr, - p_devsize, - p_size, - p_free, - p_used, - p_extent, - p_ext_alloc, - ) = line.strip().split('~') - # if the PV is not used, set VG to None - if not len(p_vg): - p_vg = None - else: - p_vg_obj = self._vgs[p_vg] - # create PV object - pv = self._PV(self, p_vg_obj, p_uuid, p_name, p_format, p_attr, - int(p_devsize), int(p_size), int(p_free), int(p_used), - int(p_extent), int(p_ext_alloc)) - # register obj - self._pvs[p_name] = pv - if p_vg: - self._vgs[p_vg]._pvs[p_name] = pv - - def _reload_vgs(self): - ''' - ''' - with self._mutex.write: - self._vgs = {} - # execute query command - (rc, output) = Exec.call([self.LVM, - 'vgs', - '--nosuffix', - '--noheadings', - '--units', 'b', - '--separator', '~', - '-o', ('vg_uuid,vg_name,vg_fmt,vg_attr,' - 'vg_size,vg_free,vg_extent_size,' - 'vg_extent_count,vg_free_count')], - merge_output = False) - if rc != 0: - raise LVMError('error getting list of VG') - # parse command output - for line in output[0].splitlines(): - # extract tokens - ( - v_uuid, - v_name, - v_format, - v_attr, - v_size, - v_free, - v_ext_size, - v_extent, - v_ext_free, - ) = line.strip().split('~') - # create VG object - vg = self._VG(self, v_uuid, v_name, v_format, v_attr, int(v_size), - int(v_free), int(v_ext_size), int(v_extent), - int(v_ext_free)) - # register object - self._vgs[v_name] = vg - - def _reload_lvs(self): - ''' - ''' - with self._mutex.write: - # execute query command - (rc, output) = Exec.call([self.LVM, - 'lvs', - '--nosuffix', - '--noheadings', - '--units', 'b', - '--separator', '~', - '-o', ('lv_uuid,lv_name,vg_name,lv_attr,' - 'lv_size')], - merge_output = False) - if rc != 0: - raise LVMError('error getting list of LV') - # parse command output - for line in output[0].splitlines(): - # extract tokens - ( - l_uuid, - l_name, - l_vg, - l_attr, - l_size, - ) = line.strip().split('~') - # create LV object - lv = self._LV(self, self._vgs[l_vg], l_uuid, l_name, l_attr, - l_size) - # register object - self._vgs[l_vg]._lvs[l_name] = lv - - def reload(self): - # VG must be reloaded before PV - self._reload_vgs() - self._reload_pvs() - self._reload_lvs() - - def get_pv(self, path): - ''' - ''' - with self._mutex.read: - return self._pvs[path] - - def get_vg(self, name): - ''' - ''' - with self._mutex.read: - return self._vgs[name] - - def get_lv(self, vgname, name): - ''' - ''' - with self._mutex.read: - return self._vgs[vgname]._lvs[name] - - def dm_create(self, name, table): - ''' - ''' - with self._mutex.write: - rc = Exec.silent([self.DMSETUP, - 'create', - name], - input=table) - if rc: - raise LVMError('failed to create DM device') - - def dm_remove(self, name, force=True): - ''' - ''' - with self._mutex.write: - # build command arguments - argv = [self.DMSETUP, 'remove'] - if force is True: - argv.append('--force') - argv.append(name) - # execute command - if Exec.silent(argv): - raise LVMError('failed to remove DM') - - def dm_get_table(self, name): - ''' - ''' - with self._mutex.read: - rc, output = Exec.call([self.DMSETUP, - 'table', - '--showkeys', - name], - merge_output=False) - table = str(output[0]).strip() - if rc or not len(table): - raise LVMError('failed to get DM table') - return table - - def dm_set_table(self, name, table): - ''' - ''' - with self._mutex.write: - rc1 = Exec.silent([self.DMSETUP, - 'load', - name], - input=table) - rc2 = Exec.silent([self.DMSETUP, - 'suspend', - name]) - rc3 = Exec.silent([self.DMSETUP, - 'resume', - name]) - if rc1 or rc2 or rc3: - raise LVMError('failed to change DM table') - - - class _Vol(object): - ''' - ''' - def __init__(self, mngr, vg, uuid, name, attrs): - self._mngr = mngr - self.vg = vg - self.uuid = str(uuid) - self.name = str(name) - self.attrs = str(attrs) - - - class _PV(_Vol): - ''' - ''' - def __init__(self, mngr, vg, uuid, name, format, attrs, device_size, - size, free, used, extent_total, extent_used): - super(LVM._PV, self).__init__(mngr, vg, uuid, name, attrs) - self.format = str(format) - self.device_size = int(device_size) - self.size = int(size) - self.free = int(free) - self.used = int(used) - self.extent_total = int(extent_total) - self.extent_used = int(extent_used) - - - class _VG(_Vol): - ''' - ''' - def __init__(self, mngr, uuid, name, format, attrs, size, free, - extent_size, extent_total, extent_free): - super(LVM._VG, self).__init__(mngr, self, uuid, name, attrs) - self._pvs = {} - self._lvs = {} - self.format = str(format) - self.size = int(size) - self.free = int(free) - self.extent_size = int(extent_size) - self.extent_total = int(extent_total) - self.extent_free = int(extent_free) - - def create(self, name, size): - ''' - ''' - # execute create command - if Exec.silent([LVM.LVCREATE, - '-L', '%iK' % (int(size)/1024), - '-n', name, - self.name]) != 0: - raise LVMError('error creating LV') - - def remove(self, name): - ''' - ''' - # execute create command - if Exec.silent([LVM.LVREMOVE, - '-f', - '%s/%s' % (self.name, name)]) != 0: - raise LVMError('error removing LV') - - def get_lv(self, name): - ''' - ''' - with self._mngr._mutex.read: - return self._lvs[name] - - class _LV(_Vol): - ''' - ''' - def __init__(self, mngr, vg, uuid, name, attrs, size): - super(LVM._LV, self).__init__(mngr, vg, uuid, name, attrs) - self.size = int(size) - #fixme - self.path = '/dev/%s/%s' % (self.vg.name, self.name) - - def dm_name(self): - ''' - ''' - return '%s-%s' % (self.vg.name.replace('-', '--'), - self.name.replace('-', '--')) - - def dm_table(self): - ''' - ''' - return self._mngr.dm_get_table(self.dm_name()) diff --git a/ccnode/modules/__init__.py b/ccnode/modules/__init__.py new file mode 100644 index 0000000..07fd942 --- /dev/null +++ b/ccnode/modules/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +LIST = [ + 'tags', + 'jobs', + 'execute', + 'shutdown', + 'sysinfo', +] diff --git a/ccnode/modules/execute.py b/ccnode/modules/execute.py new file mode 100644 index 0000000..0c5fa47 --- /dev/null +++ b/ccnode/modules/execute.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +from subprocess import Popen, PIPE, STDOUT +from ..kernel import BaseModule, ModuleError, Handler, pure + + +class Module(BaseModule): + + _handler_methods = { + 'execute_command' : '_h_execute_command', + } + + def init(self, manager): + + self._manager = manager + self._logger = manager.get_logger() + self._config = manager.get_configuration() + + # create and populate our own handler + self._handler = Handler(self._logger, 'execute') + for name, method in self._handler_methods.iteritems(): + self._handler.procedure_register(name, getattr(self, method)) + # register it in the core handler + self._manager.get_handler().register(self._handler) + + def destroy(self): + + # FIXME do something about running commands (wait? kill?) + + # unregister ourselves from the core handler + self._manager.get_handler().unregister(self._handler) + + @pure + def _h_execute_command(self, command): + + # do nothing when execution is disabled in configuration + if self._config.get('command_execution') is not True: + raise ModuleError('command execution is disabled by configuration') + + # synchronous execution + self._logger.info('executing command: %s', command) + output = None + try: + output = Popen(command, shell=True, bufsize=-1, stdin=PIPE, + stdout=PIPE, stderr=STDOUT).communicate()[0] + # command not found or related error + except OSError as err: + raise ModuleError('could not execute command: %s' % err) + # return output string, mix of stdout and stderr + else: + return output diff --git a/ccnode/modules/shutdown.py b/ccnode/modules/shutdown.py new file mode 100644 index 0000000..18ac5c1 --- /dev/null +++ b/ccnode/modules/shutdown.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +from subprocess import Popen, PIPE, STDOUT +from ..kernel import BaseModule, ModuleError, Handler, pure + + +class Module(BaseModule): + + _handler_methods = { + 'node_shutdown' : '_h_node_shutdown', + } + + def init(self, manager): + + self._manager = manager + self._logger = manager.get_logger() + self._config = manager.get_configuration() + + # create and populate our own handler + self._handler = Handler(self._logger, 'shutdown') + for name, method in self._handler_methods.iteritems(): + self._handler.procedure_register(name, getattr(self, method)) + # register it in the core handler + self._manager.get_handler().register(self._handler) + + def destroy(self): + + # unregister ourselves from the core handler + self._manager.get_handler().unregister(self._handler) + + @pure + def _h_node_shutdown(self, reboot=True, gracefull=True): + + self._logger.info('server requested shutdown of local host with' + ' options reboot=`%s` gracefull=`%s`', reboot, gracefull) + + # build option list for the GNU shutdown command + options = '' + if reboot is False: + options += ' -h -P' + options += '' if gracefull else ' -n' + else: + options += ' -r' + options += ' -f' if gracefull else ' -n' + options += ' 0' + + # shutdown machine + try: + Popen('/sbin/shutdown %s' % options, shell=True).communicate() + # command not found or related error + except OSError as err: + raise ModuleError('could not execute shutdown: %s' % err) diff --git a/ccnode/threading.py b/ccnode/threading.py new file mode 100644 index 0000000..2a3f395 --- /dev/null +++ b/ccnode/threading.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# +# Cloud-Control Node +# Copyright (C) 2011 SmartJog [http://www.smartjog.com] +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Authors: +# * 2011-06: Thibault VINCENT +# + +from __future__ import absolute_import +from threading import Lock + + +class RWLock(object): + # pylint: disable=R0903 + + def __init__(self): + + self._mutex = Lock() + self._writemutex = Lock() + self._readers = 0 + self._writers = 0 + self.read = self._RLock(self) + self.write = self._WLock(self) + + + class _Lock(object): + + def __init__(self, rwlock): + + self._parent = rwlock + + + class _WLock(_Lock): + + def __enter__(self): + + self.acquire() + + def __exit__(self, exc_type, exc_value, traceback): + + self.release() + + def acquire(self): + # pylint: disable=W0212 + + with self._parent._mutex: + self._parent._writers += 1 + self._parent._writemutex.acquire() + + def release(self): + # pylint: disable=W0212 + + with self._parent._mutex: + self._parent._writers -= 1 + self._parent._writemutex.release() + + + class _RLock(_Lock): + + def __enter__(self): + + self.acquire() + + def __exit__(self, exc_type, exc_value, traceback): + + self.release() + + def acquire(self): + # pylint: disable=W0212 + + self._parent._mutex.acquire() + if self._parent._writers > 0 or self._parent._readers == 0: + self._parent._mutex.release() + self._parent._writemutex.acquire() + self._parent._mutex.acquire() + self._parent._readers += 1 + self._parent._mutex.release() + + def release(self): + # pylint: disable=W0212 + + self._parent._mutex.acquire() + self._parent._readers -= 1 + if self._parent._readers == 0: + self._parent._writemutex.release() + self._parent._mutex.release() diff --git a/ccnode/utils.py b/ccnode/utils.py deleted file mode 100644 index cf46d22..0000000 --- a/ccnode/utils.py +++ /dev/null @@ -1,210 +0,0 @@ -# -*- coding: utf-8 -*- - -import os, sys, subprocess -from socket import gethostname, gethostbyname -from threading import Lock -from logging import debug - - -def enum(*sequential, **named): - ''' - ''' - enums = dict(zip(sequential, range(len(sequential))), **named) - return type('Enum', (), enums) - - -class Enum(dict): - ''' - Usage example: - - $ states = Enum(SLEEPING, CONNECTED="Conn", DISCONNECTED="Disco") - $ print states.CONNECTED - 1 - $ states.SLEEPING - 0 - $ states.CONNECTED_str - "Conn" - $ states.SLEEPING_str - None - $ 0 in states - True - $ 10 in states - False - $ s = "Conn" - $ if s in states: - $ val = states[s] - ''' - def __init__(self, *named, **valued): - ''' - ''' - # FIXME not memory optimal structures - self._items_map = {} - self._items_rmap = {} - self._items_val = [] - toks = [(name, None) for name in named] - toks.extend([i for i in valued.iteritems()]) - for key, tok in enumerate(toks): - self._items_map[tok[0]] = key - self._items_rmap[key] = tok[0] - self._items_val.append(tok[1]) - - def __contains__(self, item): - ''' - ''' - if isinstance(item, int): - return item in self._items_rmap - else: - return item in self._items_val - - def __getitem__(self, key): - ''' - ''' - if isinstance(key, int): - return self._items_val[key] - else: - for idx, val in enumerate(self._items_val): - if val == key: - return idx - raise IndexError('object not found in enum') - - def __getattr__(self, name): - ''' - ''' - if name in self._items_map: - return self._items_map[name] - else: - if name[-4:] == '_str': - n = name[:-4] - if n in self._items_map: - return self._items_val[self._items_map[n]] - raise AttributeError('not an enum member') - - def get(self, key, default=None): - ''' - ''' - try: - return self[key] - except IndexError as err: - return default - - -class RWLock(object): - ''' - ''' - def __init__(self): - ''' - ''' - self._mutex = Lock() - self._writemutex = Lock() - self._readers = 0 - self._writers = 0 - self.read = self._RLock(self) - self.write = self._WLock(self) - - - class _Lock(object): - ''' - ''' - def __init__(self, rwlock): - ''' - ''' - self._parent = rwlock - - - class _WLock(_Lock): - ''' - ''' - def __enter__(self): - ''' - ''' - self.acquire() - - def __exit__(self, exc_type, exc_value, traceback): - ''' - ''' - self.release() - - def acquire(self): - ''' - ''' - with self._parent._mutex: - self._parent._writers += 1 - self._parent._writemutex.acquire() - - def release(self): - ''' - ''' - with self._parent._mutex: - self._parent._writers -= 1 - self._parent._writemutex.release() - - - class _RLock(_Lock): - ''' - ''' - def __enter__(self): - ''' - ''' - self.acquire() - - def __exit__(self, exc_type, exc_value, traceback): - ''' - ''' - self.release() - - def acquire(self): - ''' - ''' - self._parent._mutex.acquire() - if self._parent._writers > 0 or self._parent._readers == 0: - self._parent._mutex.release() - self._parent._writemutex.acquire() - self._parent._mutex.acquire() - self._parent._readers += 1 - self._parent._mutex.release() - - def release(self): - ''' - ''' - self._parent._mutex.acquire() - self._parent._readers -= 1 - if self._parent._readers == 0: - self._parent._writemutex.release() - self._parent._mutex.release() - - -class Exec(object): - ''' - ''' - @staticmethod - def call(cmd, shell=False, input=None, capture_output=True, - merge_output=True): - ''' - ''' - if capture_output: - stdout = subprocess.PIPE - if merge_output: - stderr = subprocess.STDOUT - else: - stderr = subprocess.PIPE - else: - stdout = open(os.devnull, 'w') - stderr = stdout - debug('Exec.call: cmd=`%s`', ' '.join(cmd)) - if input is not None: - stdin = subprocess.PIPE - debug('Exec.call: input=`%s`', input) - else: - stdin = None - proc = subprocess.Popen(cmd, shell=shell, stdout=stdout, stderr=stderr, - stdin=stdin) - output = proc.communicate(input=input) - debug('Exec.call: rc=`%d` out=`%s`', proc.returncode, output) - return (proc.returncode, output) - - @staticmethod - def silent(cmd, **kwargs): - ''' - ''' - kwargs['capture_output'] = False - return Exec.call(cmd, **kwargs)[0] diff --git a/ccnode/xen.py b/ccnode/xen.py deleted file mode 100644 index 6bb9ce3..0000000 --- a/ccnode/xen.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- - -from libvirtwrapper import LibvirtHypervisor, MEGABYTE_DIV - - -class XenHypervisor(LibvirtHypervisor): - ''' - Base class of a Xen Hypervisor - ''' - _instance = None - - def __init__(self): - ''' - ''' - super(XenHypervisor, self).__init__('xen') - - def __new__(cls, *args, **kwargs): - ''' - .. note:: - We use singleton design pattern to force only a single instance - of our libvirt hypervisor handle, it's essential since we connect - with libvirt only on localhost so we must assure one single - connection to the hypervisor - ''' - if cls._instance is None: - cls._instance = super(XenHypervisor, cls).__new__(cls, *args, - **kwargs) - return cls._instance - - def get_mem(self): - ''' - ''' - mem = None - try: - data = self._lvcon_handle.getInfo()[1] - if data: - mem = data * MEGABYTE_DIV - except: - pass - return mem - - def get_mem_free(self): - ''' - ''' - return None - - def get_mem_used(self): - ''' - ''' - return None diff --git a/debian/init b/debian/init deleted file mode 100644 index db8d1ce..0000000 --- a/debian/init +++ /dev/null @@ -1,142 +0,0 @@ -#! /bin/sh - -### BEGIN INIT INFO -# Provides: cc-node -# Required-Start: $local_fs $remote_fs $network $syslog -# Required-Stop: $local_fs $remote_fs $network $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start the CloudControl node -# Description: starts the CloudControl node using start-stop-daemon -### END INIT INFO - -# Author: Antoine Millet -# Thibault VINCENT - -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="CloudControl node" -NAME=cc-node -DAEMON=/usr/bin/cc-node -PIDFILE=/var/run/$NAME.pid -DAEMON_OPTS="-d -p $PIDFILE" - -# Defaults: -USER=root -GROUP=root - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Load various rcS variables -. /lib/init/vars.sh - -# Override the VERBOSE variable so we always have feedback messages -VERBOSE=yes - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -# Read and parse configuration variable file if it is present -[ -r /etc/default/$NAME ] && { - . /etc/default/$NAME - # Do not start if service is disabled - if [ "$ENABLED" != "true" ] ; then - echo "$DESC disabled, see /etc/default/$NAME" - exit 0 - fi -} - -# -# Function that starts the daemon/service -# -do_start() -{ - start-stop-daemon --start --quiet --name $NAME --test \ - --exec $(readlink -f $(which python)) > /dev/null || return 2 - start-stop-daemon --start --quiet --pidfile $PIDFILE --chuid $USER:$GROUP \ - --exec $DAEMON -- $DAEMON_OPTS || return 1 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 \ - --pidfile $PIDFILE --name $NAME - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 \ - --exec $DAEMON - [ "$?" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE \ - --name $NAME - return 0 -} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - status) - status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - [ "$VERBOSE" != no ] && log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 1) [ "$VERBOSE" != no ] && log_end_msg 1 ;; # Old process is still running - *) [ "$VERBOSE" != no ] && log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - [ "$VERBOSE" != no ] && log_end_msg 1 - ;; - esac - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 5f4560d..0000000 --- a/doc/Makefile +++ /dev/null @@ -1,89 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/cc-node.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/cc-node.qhc" - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/doc/api.rst b/doc/api.rst deleted file mode 100644 index 1e2c574..0000000 --- a/doc/api.rst +++ /dev/null @@ -1,52 +0,0 @@ -Node library API -================ - -.. contents:: Table of contents - -.. automodule:: ccnode - -Node Core -_________ - -.. automodule:: ccnode.ccnode - :members: -.. automodule:: ccnode.handlers - :members: - -Common Host and Storage implementation -______________________________________ - -.. automodule:: ccnode.common - :members: - -Libvirt Host and Storage wrappers -_________________________________ - -.. automodule:: ccnode.libvirtwrapper - :members: - -KVM specific Hypervisor Node implementation -___________________________________________ - -.. automodule:: ccnode.kvm - :members: -.. automethod:: ccnode.kvm.KvmHypervisor.__new__() - -Xen specific Hypervisor Node implementation -___________________________________________ - -.. automodule:: ccnode.xen - :members: -.. automethod:: ccnode.xen.XenHypervisor.__new__() - -Exceptions -__________ - -.. automodule:: ccnode.exceptions - :members: - -Helper Functions -________________ - -.. autofunction:: ccnode.libvirtwrapper.find_process_id - diff --git a/doc/conf.py b/doc/conf.py deleted file mode 100644 index 1d82e97..0000000 --- a/doc/conf.py +++ /dev/null @@ -1,205 +0,0 @@ -# -*- coding: utf-8 -*- -# -# cc-node documentation build configuration file, created by -# sphinx-quickstart on Mon Dec 20 17:27:00 2010. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys, os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('../')) -sys.path.append(os.path.abspath('../../cc-server')) -sys.path.append(os.path.abspath('../../sjrpc')) - -# -- General configuration ----------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', -'sphinx.ext.todo', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', -'sphinx.ext.graphviz', 'sphinx.ext.doctest'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'cc-node' -copyright = u'2010, Smartjog' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -from ccnode import __version__ -version = __version__ -# The full version, including alpha/beta/rc tags. -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -#unused_docs = [] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. Major themes that come with -# Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'default' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { - 'stickysidebar' : 'true' - } - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_use_modindex = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'cc-nodedoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'cc-node.tex', u'cc-node Documentation', - u'Smartjog', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_use_modindex = True - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/index.rst b/doc/index.rst deleted file mode 100644 index 8e5d024..0000000 --- a/doc/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. cc-node documentation master file, created by - sphinx-quickstart on Mon Dec 20 17:27:00 2010. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to cc-node's documentation! -=================================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - api - libvirt - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/doc/libvirt.rst b/doc/libvirt.rst deleted file mode 100644 index 82a13ed..0000000 --- a/doc/libvirt.rst +++ /dev/null @@ -1,43 +0,0 @@ -Libvirt Virtualization API -========================== - -.. contents:: Table of contents - -.. automodule:: libvirt - -Connection -__________ - -.. autoclass:: libvirt.virConnect - :members: - -Domains -_______ - -.. autoclass:: libvirt.virDomain - :members: - -Storage Pools -_____________ - -.. autoclass:: libvirt.virStoragePool - :members: - -Storage Volumes -_______________ - -.. autoclass:: libvirt.virStorageVol - :members: - -Network -_______ - -.. autoclass:: libvirt.virNetwork - :members: - -Links -_____ - -Official libvirt API: `link`_ . - -.. _link: http://libvirt.org/html/index.html diff --git a/etc/cc-node.conf b/etc/cc-node.conf old mode 100644 new mode 100755 index 5e397f1..90d49f7 --- a/etc/cc-node.conf +++ b/etc/cc-node.conf @@ -1,23 +1,7 @@ [node] - -# address and port of the CloudControl server -address = 10.15.255.42 -#port = 1984 - -# account created for this node on the server -login = $$LOGIN$$ -password = $$PASSWORD$$ - -# logging verbosity level 0-3 -#verbosity = 0 - -# hypervisor detection and connection, set to 'no' on regular hosts or when -# the node should not attempt to use local hypervisors -#detect_hypervisor = yes - -# allow remote command execution (or host shutdown/reboot) -#command_execution = yes - -# TO BE REMOVED IN LATER RELEASES -# force usage of Xen -#force_xen = no \ No newline at end of file +address = cc-server.lab.fr.lan +login = tvincent_kvm +password = toto +verbosity = 3 +virtualization = no +command_execution = yes -- GitLab