Commit 2c715413 authored by Antoine Millet's avatar Antoine Millet

Initial import

parents
*.pyc
doc/_build/*
*.swp
*.log
test_*.py
This diff is collapsed.
recursive-include etc *
#!/usr/bin/env python
#coding=utf8
import sys
import logging
import logging.handlers
import ConfigParser
import signal
from optparse import OptionParser
from ccserver.ccserver import CCServer
import os
try:
import daemon
DAEMONIZE = True
except ImportError:
DAEMONIZE = False
__VERSION__ = 3
DEFAULT_CONFIG_FILE = '/etc/cc-server.conf'
DEFAULT_PID_FILE = '/var/run/cc-server.pid'
DEFAULT_CONFIGURATION = {
'port': 1984,
'verbosity': 0,
'account_db': None, # None = mandatory option
'interface': '127.0.0.1',
'ssl_cert': None,
'ssl_key': None,
}
def run_server(options):
# Setup logging facility:
level = logging.ERROR
verb = int(options['verbosity'])
if verb:
if verb == 1:
level = logging.WARNING
elif verb == 2:
level = logging.INFO
else:
level = logging.DEBUG
logger = logging.getLogger()
logger.setLevel(level)
facility = logging.handlers.SysLogHandler.LOG_DAEMON
handler = logging.handlers.SysLogHandler(address='/dev/log',
facility=facility)
fmt = logging.Formatter('cc-server: %(levelname)s %(message)s')
handler.setFormatter(fmt)
logger.addHandler(handler)
server = CCServer(conf_dir=options['account_db'], port=int(options['port']),
address=options['interface'], keyfile=options['ssl_key'],
certfile=options['ssl_cert'])
def shutdown_handler(signum, frame):
'''
Handler called when SIGINT is emitted.
'''
server.manager.shutdown()
logging.info('Server properly exited by SIGINT')
sys.exit(0)
signal.signal(signal.SIGINT, shutdown_handler)
try:
server.run()
except Exception as err:
import traceback
traceback.print_exc(file=sys.stdout)
logging.critical('Server fail: %s' % err)
sys.exit(3)
if __name__ == '__main__':
op = OptionParser(version='%%prog %s' % __VERSION__)
op.add_option('-c', '--config', default=DEFAULT_CONFIG_FILE,
help='configuration file (default: %default)')
op.add_option('-d', '--daemonize', default=False, action='store_true',
help='run as daemon and write pid file')
op.add_option('-p', '--pid-file', default=DEFAULT_PID_FILE,
help='pid file (default: %default)')
cliopts, args = op.parse_args()
# Reading the config file:
config = ConfigParser.SafeConfigParser()
config.read(cliopts.config)
try:
options = dict(config.items('server'))
except ConfigParser.NoSectionError:
sys.stderr.write("Configuration error: 'server' section must exist "
"in '%s'\n" % cliopts.config)
sys.exit(1)
# Applying default config file options:
for opt, default in DEFAULT_CONFIGURATION.iteritems():
if opt not in options or not options[opt]:
if default is None:
sys.stderr.write("Configuration error: you must specify '%s' "
"option in '%s' !\n" % (opt, cliopts.config))
sys.exit(1)
else:
options[opt] = default
if cliopts.daemonize:
if DAEMONIZE:
with daemon.DaemonContext(stderr=sys.stderr):
run_server(options)
else:
sys.stderr.write(('You must install python-daemon to handle the '
'daemonization of the process.\n'))
sys.exit(2)
else:
run_server(options)
#!/usr/bin/env python
#coding=utf8
'''
CloudControl server libraries.
'''
#!/usr/bin/env python
#coding=utf8
import logging
import socket
from sjrpc.server import SimpleSslRpcServer
from handlers import WelcomeHandler
from conf import CCConf
from client import CCClient
class CCServer(object):
'''
CloudControl server main class.
:param conf_dir: the directory that store the client configuration
:param certfile: the path to the ssl certificate
:param keyfile: the path to the ssl key
:param address: the interface to bind
:param port: the port to bind
'''
LISTEN_BACKLOG = 5
def __init__(self, conf_dir, certfile=None, keyfile=None,
address='0.0.0.0', port=1984):
# Dict containing all connected accounts, the key is the login of
# the account and the value the :class:`RpcConnection` of the peer:
self._connected = {}
# The interface object to the configuration directory:
self.conf = CCConf(conf_dir)
# Create the server socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((address, port))
sock.listen(CCServer.LISTEN_BACKLOG)
if certfile:
logging.info('SSL Certificate: %s' % certfile)
if keyfile:
logging.info('SSL Key: %s' % certfile)
logging.info('Started to listen on %s port %s' % (address, port))
# Create the connection manager:
self.manager = SimpleSslRpcServer(sock, certfile=certfile,
keyfile=keyfile,
default_handler=WelcomeHandler(self))
def iterrole(self, role):
'''
Generator to iter on each connected client with specified role.
:param role: role to filter
'''
for login, client in self._connected.iteritems():
if client.connection.get_handler().__class__.role_name == role:
yield client
def register(self, login, role, connection):
'''
Register a new connected account on the server.
:param login: login of the account
:param connection: connection to register
:param tags: tags to add for the client
'''
self._connected[login] = CCClient(login, role, self, connection)
def run(self):
'''
Run the server mainloop.
'''
logging.info('Running manager mainloop')
self.manager.run()
#!/usr/bin/env python
#coding=utf8
class CCClient(object):
'''
Represent a single client connected to the server.
'''
def __init__(self, login, role, server, connection):
# The login of the client:
self._login = login
# The role of the client:
self._role = role
# The server binded to this client:
self._server = server
# The connection of the client (public attribute):
self.connection = connection
def get_tags(self, tags=None):
'''
Get the tags of the object.
:param tags: the list of tags to fetch, or None for all
'''
# Get the tags from the server
tags = self.connection.call('get_tags', tags)
# Add the static tags:
tags['a'] = self._login
tags['hv'] = self._login
tags['role'] = self._role
# Update it with the locally defined tags:
tags.update(self._server.conf.show(self._login)['tags'])
return tags
#!/usr/bin/env python
#coding=utf8
'''
This module provide an abstraction to the clients configuration directory.
The client configuration directory contains a list of ``.json`` files, each
file contains the configuration for one client. The username of the client is
the filename (excluding the extension).
The schema of the json file is described below::
{
'password': '<client password>',
'role': '<node|client>',
'tags': ['tag1', 'tag2'],
'perms': Null
}
Usage example:
>>> conf = CCConf('/etc/cloudcontrol/clients')
>>> conf.create_account(login='rms', password='secret', role='client')
>>> conf.create_account(login='server42', password='secret', role='node')
>>> print conf.authentify('server42', 'pouet')
None
>>> print conf.authentify('server42', 'secret')
u'node'
>>> conf.add_tag('rms', 'admin')
>>> conf.show('rms')
{'password': 'secret'
'role': 'client',
'tags': ['admin'],
'perms': None}
>>> conf.remove_account('rms')
>>>
'''
import threading
import logging
import json
import os
from functools import wraps
class CCConf(object):
'''
Create a new configuration interface.
:param path_directory: the directory to store the configuration files
'''
CONF_TEMPLATE = {'password': None,
'type': None,
'tags': [],
'perms': None}
def __init__(self, path_directory):
self._path = path_directory
self._lock = threading.Lock()
def _writer(func):
'''
Decorator used to threadsafize methods that made write operations on
client configuration tree.
'''
@wraps(func)
def f(self, *args, **kwargs):
with self._lock:
return func(self, *args, **kwargs)
return f
def _get_conf(self, login):
'''
Return the configuration of a client by its login.
:param login: login of the client
:return: the configuration of the client
:raise CCConf.UnknownAccount: if user login is unknown
'''
filename = os.path.join(self._path, '%s.json' % login)
if os.path.isfile(filename):
conf = json.load(open(filename, 'r'))
logging.debug('Getting configuration %s: %s' % (filename, conf))
return conf
else:
raise CCConf.UnknownAccount('%s is not a file' % filename)
def _set_conf(self, login, conf, create=False):
'''
Update the configuration of a client by its login.
:param login: login of the client
:param conf: configuration to set for the client
:raise CCConf.UnknownAccount: if user login is unknown
'''
filename = os.path.join(self._path, '%s.json' % login)
logging.debug('Writing configuration %s:' % (filename, conf))
if os.path.isfile(filename) ^ create:
json.dump(conf, open(filename, 'w'))
else:
raise CCConf.UnknownAccount('%s is not a file' % filename)
def show(self, login):
'''
Show the configuration for specified account.
:param login: the login of the client
:return: configuration of user
'''
return self._get_conf(login)
def authentify(self, login, password):
'''
Authentify the client providing its login and password. The function
return the role of the client on success, or ``None``.
:param login: the login of the client
:param password: the password of the client
:return: the client's role or None on failed authentication
:raise CCConf.UnknownAccount: if user login is unknown
'''
conf = self._get_conf(login)
if conf['password'] == password:
return conf['role']
else:
return None
@_writer
def set_password(self, login, password):
'''
Update the client's password in the configuration.
:param login: login of the user
:param password: new password
:raise CCConf.UnknownAccount: if user login is unknown
'''
conf = self._get_conf(login)
conf['password'] = password
self._set_conf(login, conf)
@_writer
def add_tag(self, login, tag):
'''
Add the tag to the user.
:param login: login of the user
:param tag: tag to add to the user
:raise CCConf.UnknownAccount: if user login is unknown
'''
logging.debug('Added tag %s for %s account' % (login, tag))
conf = self._get_conf(login)
tags = set(conf['tags'])
tags.add(tag)
conf['tags'] = list(tags)
self._set_conf(login, conf)
@_writer
def remove_tag(self, login, tag):
'''
Remove the tag to the user.
:param login: login of the user
:param tag: tag to remove to the user
:raise CCConf.UnknownAccount: if user login is unknown
'''
logging.debug('Removed tag %s for %s account' % (login, tag))
conf = self._get_conf(login)
tags = set(conf['tags'])
tags.remove(tag)
conf['tags'] = list(tags)
self._set_conf(login, conf)
@_writer
def remove_account(self, login):
'''
Remove the configuration of the account.
:param login: login of the account to remove
:raise CCConf.UnknownAccount: if user login is unknown
'''
logging.debug('Removed %s account' % login)
filename = os.path.join(self._path, '%s.json' % login)
if os.path.exists(filename):
os.remove(filename)
else:
raise CCConf.UnknownAccount('%s is not a file' % filename)
@_writer
def create_account(self, login, password, role):
'''
Create a new account.
:param login: login of the new user
:param password: password of the new user
:raise CCConf.AlreadyExistingAccount: if the login is already
'''
logging.debug('Creating %s account with role %s' % (login, role))
filename = os.path.join(self._path, '%s.json' % login)
if os.path.exists(filename):
raise CCConf.AlreadyExistingAccount('%s found' % filename)
else:
conf = CCConf.CONF_TEMPLATE.copy()
conf['password'] = password
conf['role'] = role
self._set_conf(login, conf, create=True)
class UnknownAccount(Exception):
pass
class AlreadyExistingAccount(Exception):
pass
#!/usr/bin/env python
#coding=utf8
import inspect
import logging
from sjrpc.utils import RpcHandler, pure
from tql import TqlQuery
def listed(func):
func.__listed__ = True
return func
class CCHandler(RpcHandler):
'''
Base class for handlers of CloudControl server.
:cvar role_name: the name of the role binded to the handler
:type role_name: str or None is not applicable
'''
role_name = None
def __init__(self, server):
self._server = server
def __getitem__(self, name):
logging.debug('Called method %s.%s' % (self.__class__.__name__, name))
return super(CCHandler, self).__getitem__(name)
@pure
def list_commands(self):
cmd_list = []
for attr in dir(self):
attr = getattr(self, attr, None)
if getattr(attr, '__listed__', False):
cmd = {}
cmd['name'] = attr.__name__
cmd['description'] = inspect.cleandoc(inspect.getdoc(attr))
cmd_list.append(cmd)
return cmd_list
class HypervisorHandler(CCHandler):
'''
Handler binded to 'node' role.
'''
role_name = 'hypervisor'
class ClientHandler(CCHandler):
'''
Handler binded to 'cli' role.
'''
role_name = 'client'
@pure
@listed
def list_vm(self):
'''
List all VMs on nodes.
'''
found_vm = []
async_ids = set()
# Send the asynchronous request to all nodes:
for node in self._server.iterrole('node'):
async_ids.add(node.async_call('list_vm'))
# Wait for the response:
responses = self._server.manager.wait(frozenset(async_ids), timeout=5)
# Process the responses:
for resp in responses:
if resp['error'] is None:
for vm in resp['return']:
found_vm.append(vm)
return tuple(found_vm)
@pure
@listed
def list(self, query):
'''
List all objects registered on this instance.
'''
logging.debug('Executed list function with query %s' % query)
# Contains all the objects to be filted:
objects = []
# The query object:
query = TqlQuery(query)
# Get the hypervisors that match the query:
hypervisors = list(self._server.iterrole('hypervisor'))
if query.has_condition('hv'):
hypervisors = query.filter(hypervisors, tag_name='hv',
key=lambda o,n: o.get_tags([n]).get(n))
logging.debug('Querying %d hypervisors: %s' % (len(hypervisors),
hypervisors))
# Try to get vm for each matched hypervisor:
async_calls = {}
for hy in hypervisors:
async_calls[hy.connection.async_call('list_vm')] = hy
logging.debug('Waiting for the response of hypervisors...')
responses = self._server.manager.wait(frozenset(async_calls), timeout=5)
logging.debug('Responses received.')
# Make the hypervisor object -> tags mapping:
hv_tags = {}
for hy in hypervisors:
tags = hy.get_tags(tags=tuple(query.tags))
hv_tags[hy] = tags
objects.append(tags)
for resp in responses:
if resp['error'] is None:
hy = async_calls[resp['id']]
for vm in resp['return']:
vm_tags = hv_tags[hy].copy()
vm_tags.update(vm)
vm_tags['role'] = 'vm'
objects.append(vm_tags)
# Filter the objects with the query, and return it:
return query.filter(objects)
class WelcomeHandler(CCHandler):
'''
Default handler used on client connections of the server.
:cvar ROLES: role name/handler mapping
'''
ROLES = {
'client': ClientHandler,
'hypervisor': HypervisorHandler,
}
@listed
def authentify(self, connection, login, password):
'''
Authenticate the client.
'''
role = self._server.conf.authentify(login, password)
if role is None:
logging.info('New authentication from %s: failure' % login)
return False
else:
# If authentication is a success, ask tags to the server:
self._server.register(login, role, connection)
connection.set_handler(WelcomeHandler.ROLES.get(role)(self._server))
logging.info('New authentication from %s: success' % login)
return role
#!/usr/bin/env python
#coding=utf8
import re
from fnmatch import fnmatch
MULTIPLICATOR_TABLE = {'B': 1, 'b': 1, 'o': 1,
'K': 100, 'k': 100,
'M': 1000, 'm': 1000,
'G': 10000, 'g': 10000,
'T': 100000, 't': 100000}
def gen_all_operators(operators):
ops = []
ops += list(operators)
ops += ['!' + o for o in operators]
return '|'.join(ops)
def multiplicator(number):
number = str(number)
try:
if len(number) > 1 and number[-1] in MULTIPLICATOR_TABLE:
integer = int(number[:-1]) * MULTIPLICATOR_TABLE[number[-1]]
else:
integer = int(number)
except ValueError:
return 0
return integer
class TqlParsingError(Exception):
pass
class TqlCondition(object):
'''
Represent a single condition for a tag (eg: value of mem eq 1000G).
:param name: the name of the tag
:param value: the matching value
:param operator: the operator function
'''
def __init__(self, name, value, operator, invert=False):
self.name = name
self.value = value
self.operator = operator
self.invert = invert
def __hash__(self):
return hash((self.name, self.value, self.operator, self.invert))
def __repr__(self):
return '<TqlCondition object for %s value %s>' % (self.name, self.value)
class TqlQuery(object):
'''
Parse a query written with TQL (Tag Query Language) and allow to filter a
liste of tagged objects.
:param query: the query to parse
'''
OPERATORS = {':': 'glob',
'=': 'equal',
'>': 'gt',
'<': 'lt',
'>=': 'gte',
'<=': 'lte',
'~': 'regex'}
REGEX = re.compile(r'^(?P<name>[a-z-A-Z0-9_-]+)'
r'((?P<operator>[!$<>~:=]{1,3})'
r'(?P<value>[^&]+))?$')
def __init__(self, query):
self._conditions = set()
self._parse(query)
def _parse(self, query):
'''
Parse the TQG query and fill the condition set.
:param query: the query to parse
'''
conditions = query.split('&')
for cond in conditions:
m = TqlQuery.REGEX.match(cond)
if not m:
raise TqlParsingError(("Error while parsing, invalid '"
"condition: '%s'") % cond)
# Retrieve each parts:
name = m.group('name')