pax_global_header 0000666 0000000 0000000 00000000064 12600735274 0014520 g ustar 00root root 0000000 0000000 52 comment=a8b3e7f86af38e4c69e2ebe267929c2cdf7057e0
cc-common-v8/ 0000775 0000000 0000000 00000000000 12600735274 0013272 5 ustar 00root root 0000000 0000000 cc-common-v8/.gitignore 0000664 0000000 0000000 00000000066 12600735274 0015264 0 ustar 00root root 0000000 0000000 *.pyc
doc/_build/*
*.swp
*.log
test_*.py
.ropeproject
cc-common-v8/COPYRIGHT 0000664 0000000 0000000 00000000033 12600735274 0014561 0 ustar 00root root 0000000 0000000 Copytight © 2012 Smartjog
cc-common-v8/LICENSE 0000664 0000000 0000000 00000016743 12600735274 0014312 0 ustar 00root root 0000000 0000000 GNU LESSER GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc.
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
This version of the GNU Lesser General Public License incorporates
the terms and conditions of version 3 of the GNU General Public
License, supplemented by the additional permissions listed below.
0. Additional Definitions.
As used herein, "this License" refers to version 3 of the GNU Lesser
General Public License, and the "GNU GPL" refers to version 3 of the GNU
General Public License.
"The Library" refers to a covered work governed by this License,
other than an Application or a Combined Work as defined below.
An "Application" is any work that makes use of an interface provided
by the Library, but which is not otherwise based on the Library.
Defining a subclass of a class defined by the Library is deemed a mode
of using an interface provided by the Library.
A "Combined Work" is a work produced by combining or linking an
Application with the Library. The particular version of the Library
with which the Combined Work was made is also called the "Linked
Version".
The "Minimal Corresponding Source" for a Combined Work means the
Corresponding Source for the Combined Work, excluding any source code
for portions of the Combined Work that, considered in isolation, are
based on the Application, and not on the Linked Version.
The "Corresponding Application Code" for a Combined Work means the
object code and/or source code for the Application, including any data
and utility programs needed for reproducing the Combined Work from the
Application, but excluding the System Libraries of the Combined Work.
1. Exception to Section 3 of the GNU GPL.
You may convey a covered work under sections 3 and 4 of this License
without being bound by section 3 of the GNU GPL.
2. Conveying Modified Versions.
If you modify a copy of the Library, and, in your modifications, a
facility refers to a function or data to be supplied by an Application
that uses the facility (other than as an argument passed when the
facility is invoked), then you may convey a copy of the modified
version:
a) under this License, provided that you make a good faith effort to
ensure that, in the event an Application does not supply the
function or data, the facility still operates, and performs
whatever part of its purpose remains meaningful, or
b) under the GNU GPL, with none of the additional permissions of
this License applicable to that copy.
3. Object Code Incorporating Material from Library Header Files.
The object code form of an Application may incorporate material from
a header file that is part of the Library. You may convey such object
code under terms of your choice, provided that, if the incorporated
material is not limited to numerical parameters, data structure
layouts and accessors, or small macros, inline functions and templates
(ten or fewer lines in length), you do both of the following:
a) Give prominent notice with each copy of the object code that the
Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the object code with a copy of the GNU GPL and this license
document.
4. Combined Works.
You may convey a Combined Work under terms of your choice that,
taken together, effectively do not restrict modification of the
portions of the Library contained in the Combined Work and reverse
engineering for debugging such modifications, if you also do each of
the following:
a) Give prominent notice with each copy of the Combined Work that
the Library is used in it and that the Library and its use are
covered by this License.
b) Accompany the Combined Work with a copy of the GNU GPL and this license
document.
c) For a Combined Work that displays copyright notices during
execution, include the copyright notice for the Library among
these notices, as well as a reference directing the user to the
copies of the GNU GPL and this license document.
d) Do one of the following:
0) Convey the Minimal Corresponding Source under the terms of this
License, and the Corresponding Application Code in a form
suitable for, and under terms that permit, the user to
recombine or relink the Application with a modified version of
the Linked Version to produce a modified Combined Work, in the
manner specified by section 6 of the GNU GPL for conveying
Corresponding Source.
1) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (a) uses at run time
a copy of the Library already present on the user's computer
system, and (b) will operate properly with a modified version
of the Library that is interface-compatible with the Linked
Version.
e) Provide Installation Information, but only if you would otherwise
be required to provide such information under section 6 of the
GNU GPL, and only to the extent that such information is
necessary to install and execute a modified version of the
Combined Work produced by recombining or relinking the
Application with a modified version of the Linked Version. (If
you use option 4d0, the Installation Information must accompany
the Minimal Corresponding Source and Corresponding Application
Code. If you use option 4d1, you must provide the Installation
Information in the manner specified by section 6 of the GNU GPL
for conveying Corresponding Source.)
5. Combined Libraries.
You may place library facilities that are a work based on the
Library side by side in a single library together with other library
facilities that are not Applications and are not covered by this
License, and convey such a combined library under terms of your
choice, if you do both of the following:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities,
conveyed under the terms of this License.
b) Give prominent notice with the combined library that part of it
is a work based on the Library, and explaining where to find the
accompanying uncombined form of the same work.
6. Revised Versions of the GNU Lesser General Public License.
The Free Software Foundation may publish revised and/or new versions
of the GNU Lesser General Public License from time to time. Such new
versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the
Library as you received it specifies that a certain numbered version
of the GNU Lesser General Public License "or any later version"
applies to it, you have the option of following the terms and
conditions either of that published version or of any later version
published by the Free Software Foundation. If the Library as you
received it does not specify a version number of the GNU Lesser
General Public License, you may choose any version of the GNU Lesser
General Public License ever published by the Free Software Foundation.
If the Library as you received it specifies that a proxy can decide
whether future versions of the GNU Lesser General Public License shall
apply, that proxy's public statement of acceptance of any version is
permanent authorization for you to choose that version for the
Library.
cc-common-v8/MANIFEST.in 0000664 0000000 0000000 00000000017 12600735274 0015026 0 ustar 00root root 0000000 0000000 include README
cc-common-v8/README 0000664 0000000 0000000 00000000074 12600735274 0014153 0 ustar 00root root 0000000 0000000 CloudControl Common Libraries
=============================
cc-common-v8/cloudcontrol/ 0000775 0000000 0000000 00000000000 12600735274 0016001 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/__init__.py 0000664 0000000 0000000 00000001360 12600735274 0020112 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
__import__('pkg_resources').declare_namespace(__name__)
cc-common-v8/cloudcontrol/common/ 0000775 0000000 0000000 00000000000 12600735274 0017271 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/__init__.py 0000664 0000000 0000000 00000001346 12600735274 0021406 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" CloudControl common libraries package.
"""
cc-common-v8/cloudcontrol/common/allocation/ 0000775 0000000 0000000 00000000000 12600735274 0021416 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/allocation/__init__.py 0000664 0000000 0000000 00000001265 12600735274 0023533 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see . cc-common-v8/cloudcontrol/common/allocation/vmspec.py 0000664 0000000 0000000 00000014260 12600735274 0023270 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
import re
class VMSpecValidationError(Exception):
""" Exception raised when a VMSpec validation error occurs.
"""
# Checks:
def check_type(object_, type_, message, **kwargs):
if not isinstance(object_, type_):
kwargs['type'] = type_.__class__.__name__
kwargs['object'] = str(object_)
raise VMSpecValidationError(message % kwargs)
def check_regex(object_, regex, message, **kwargs):
if re.match(object_, regex):
kwargs['object'] = str(object_)
raise VMSpecValidationError(message % kwargs)
def check_integer(object_, message, min=None, max=None, **kwargs):
if not isinstance(object_, int):
raise VMSpecValidationError(message % kwargs)
elif (min is not None and object_ < min) or (max is not None and object_ > max):
raise VMSpecValidationError(message % kwargs)
def check_dict_key(dict_, key, message, **kwargs):
if key not in dict_:
kwargs['key'] = key
raise VMSpecValidationError(message % kwargs)
# Validation rules:
def validate_riskgroup(data):
""" Validate riskgroups.
Example:
{'name': {'tag': 1}}
"""
check_type(data, dict, 'riskgroup: must be a dict')
for name, tags in data.iteritems():
env = {"name": name}
check_type(name, basestring, 'riskgroup/[%(name)s]: must be a string', **env)
check_type(tags, dict, 'riskgroup/%(name)s: must be a dict', **env)
for tag, value in tags.iteritems():
check_type(tag, basestring, 'riskgroup/%(name)s/[%(tag)s]: must be a string', tag=tag, **env)
check_integer(value, 'riskgroup/%(name)s/%(tag)s: must be a positive integer', min=1, tag=tag, **env)
def validate_machine_volumes(hostname, data):
""" Validate machines volumes.
Example:
{'root': {'size': 512000, 'pool': 'vg'}}
"""
check_type(data, list, 'machines/%(hostname)s/volumes: must be a list', hostname=hostname)
for i, disk in enumerate(data):
env = {'hostname': hostname, 'index': i}
check_type(disk, dict, 'machines/%(hostname)s/volumes/%(index)s: must be a %(type)s', **env)
check_dict_key(disk, 'size', 'machines/%(hostname)s/volumes/%(index)s/[%(key)s]: is mandatory', **env)
check_integer(disk['size'], 'machines/%(hostname)ss/volumes/%(index)s/size: must be a positive integer', min=0, **env)
check_dict_key(disk, 'pool', 'machines/%(hostname)s/volumes/%(index)s/[%(key)s]: is mandatory', **env)
check_type(disk['pool'], basestring, 'machines/%(hostname)ss/volumes/%(index)s/pool: must be a string', **env)
def validate_machine_spec(hostname, spec):
""" Validate machines specifications.
Example:
{'cpu': 8,
'memory': 512000,
'flags': ['does_not_autostart'],
'tags': {'platform': 'Infra'}}
"""
env = {'hostname': hostname}
check_integer(spec.get('cpu'), 'machines/%(hostname)s/cpu: must be a positive integer', min=1, **env)
check_integer(spec.get('memory'), 'machines/%(hostname)s/memory: must be a positive integer', min=1, **env)
if 'flags' in spec:
if spec['flags'] is None:
spec['flags'] = []
check_type(spec['flags'], list, 'machines/%(hostname)s/flags: must be a list', **env)
for i, flag in enumerate(spec['flags']):
check_type(flag, basestring, 'machines/%(hostname)s/flags[%(i)s]: must be a string', i=i, **env)
if 'tags' in spec:
if spec['tags'] is None:
spec['tags'] = {}
check_type(spec['tags'], dict, 'machines/%(hostname)s/tags: must be a dict', **env)
for tag, value in spec['tags'].iteritems():
check_type(tag, basestring, 'machines/%(hostname)s/tags/[%(tag)s]: must be a string', tag=tag, **env)
check_type(value, basestring, 'machines/%(hostname)s/tags/%(tag)s: must be a string', tag=tag, **env)
if 'volumes' in spec:
validate_machine_volumes(hostname, spec['volumes'])
def validate_machines(data):
""" Validate machine entry.
Example:
{'foobar-1.example.org': ...}
"""
check_type(data, dict, 'machines: must be a dict')
if len(data) < 1:
raise VMSpecValidationError('machines: at least one machine must be defined')
for hostname, spec in data.iteritems():
if not re.match('^[a-z0-9-]+(.[a-z0-9-]+)*$', hostname):
raise VMSpecValidationError('machines/[%s]: bad hostname format' % hostname)
validate_machine_spec(hostname, spec)
def validate_vmspec(data):
""" Validate a vmspec.
Example:
{'riskgroups:' ..., 'machines': ...}
"""
if 'riskgroups' in data:
validate_riskgroup(data['riskgroups'])
check_dict_key(data, 'machines', 'machines: is mandatory')
validate_machines(data['machines'])
if 'target' in data:
check_type(data['target'], basestring, 'target: must be a string (a TQL)')
def expand_vmspec(vmspec):
""" Expand the vmspec to a list of VM.
"""
validate_vmspec(vmspec)
vmspec_expanded = []
riskgroups = vmspec.get('riskgroups', {})
for title, specs in vmspec['machines'].iteritems():
specs['title'] = title
riskgroup = specs.get('tags', {}).get('riskgroup')
if riskgroup is not None:
if riskgroup not in riskgroups:
raise VMSpecValidationError('machines/%s: riskgroup not defined')
specs['riskgroup'] = riskgroups[riskgroup]
vmspec_expanded.append(specs)
return vmspec_expanded
if __name__ == '__main__':
import sys, json
validate_vmspec(json.load(sys.stdin))
cc-common-v8/cloudcontrol/common/client/ 0000775 0000000 0000000 00000000000 12600735274 0020547 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/client/__init__.py 0000664 0000000 0000000 00000001266 12600735274 0022665 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
cc-common-v8/cloudcontrol/common/client/exc.py 0000664 0000000 0000000 00000002113 12600735274 0021675 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
class CloudControlError(Exception):
"""Base class for cloud control errors."""
class PluginError(CloudControlError):
"""Exception related to plugin execution."""
pass
class TagConflict(CloudControlError):
"""Raised when a tag name conflicts arises between TagDB instances."""
pass
class ConfigError(CloudControlError):
"""Raised in case of configuration error."""
pass
cc-common-v8/cloudcontrol/common/client/loop.py 0000664 0000000 0000000 00000050330 12600735274 0022073 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
import os
import sys
import ssl
import time
import Queue
import errno
import signal
import socket
import threading
import logging
import logging.config
from functools import partial
import pyev
from sjrpc.core import RpcConnection, RpcError
from sjrpc.utils import threadless
from cloudcontrol.common.client.tags import get_tags, RootTagDB
from cloudcontrol.common.client.exc import PluginError, ConfigError
logger = logging.getLogger(__name__)
class RPCStartHandler(object):
"""Handles rpc connection and authentication to the remote cc-server.
It performs the following steps (modelized as a DFA):
| Name service resolution
->
| TCP connection
->
| SSL handshake
->
| authentication to the cc-server
->
| done
"""
AUTH_TIMEOUT = 30.
def __init__(self, loop):
"""
:param loop: MainLoop instance
"""
# internal DFA state handling
self._states = {
# FORMAT IS: STATE: (INIT, CLEANUP)
# INIT is a pre state execution that is called to setup things like
# watchers, threads
# CLEANUP is a post state that cleans what is set in the INIT
self.handle_name: (self.init_name, self.cleanup_name),
self.handle_connect: (self.init_connect, self.cleanup_connect),
self.handle_handshake: (self.init_handshake, self.cleanup_handshake),
self.handle_authentication: (self.init_authentication,
self.cleanup_authentication),
self.handle_done: (None, None),
self.handle_error: (None, None), # obviously error state
}
self._current_state = None
self.loop = loop
# we don't declare all attributes in the constructor since some are only
# specified to a single state
self.sock = None
self.rpc_con = None
# id for async rpc call
self.auth_id = None
def _goto(self, where):
"""Method that is used to perform a state change, it will automatically
call init/cleanup methods if needed.
:param where: state where we wanna go or None
"""
cleanup = self._states.get(self._current_state, (None, None))[1]
if cleanup is not None:
cleanup()
self._current_state = where
if where is not None:
init = self._states[where][0]
if init is not None:
init()
where()
def init_name(self):
self.thread = threading.Thread(target=self._handle_name)
self.thread.daemon = True
self.async_w = self.loop.evloop.async(self.handle_name_cb)
def handle_name(self):
self.async_w.start()
self.thread.start()
# Method run in the thread
def _handle_name(self):
while True:
logger.debug('getaddrinfo')
try:
self.addr_info = iter(socket.getaddrinfo(
self.loop.config.server_host,
self.loop.config.server_port,
socket.AF_UNSPEC,
socket.SOCK_STREAM,
))
except socket.gaierror as exc:
logger.error(
'Error while resolving hostname for cc-server: %s',
exc.strerror,
)
except Exception:
logger.exception(
'Unexpected error while resolving cc-server hostname')
else:
break
time.sleep(5.)
self.async_w.send()
def handle_name_cb(self, watcher, revents):
self._goto(self.handle_connect)
def cleanup_name(self):
del self.thread
self.async_w.stop()
del self.async_w
def init_connect(self):
# watch for non-blocking connection
self.sock_write_w = None
# tick is used for connect -> name transition to introduce some delay
# between attempts
self.tick_w = self.loop.evloop.timer(2., 0., self.handle_connect_error)
def handle_connect_error(self, watcher, revents):
self._goto(self.handle_name)
def handle_connect(self):
while True:
try:
(
family,
type_,
proto,
cannonname,
sockaddr,
) = self.current_addr = self.addr_info.next()
except StopIteration:
logger.error('Did try all addrinfo entries for cc-server without '
'success, will try to resolve hostname again')
# we need to cleanup socket for the connect -> name transition
# only, so we don't write the code in the cleanup method
if self.sock is not None:
self.sock.close()
self.tick_w.start()
return
try:
self.sock = socket.socket(family, type_, proto)
except EnvironmentError as exc:
logger.error(
'Cannot create socket to connect to cc-server '
'(Errno %s: %s)', exc.errno, exc.strerror,
)
sys.exit(1)
return
try:
self.sock.setblocking(0)
except EnvironmentError as exc:
logger.error(
'Cannot set socket in non blocking mode (Errno %s: %s),'
' will exit', exc.errno, exc.strerror,
)
sys.exit(1)
return
error = self.sock.connect_ex(sockaddr)
if error == errno.EINPROGRESS: # most likely to happen
self.sock_write_w = self.loop.evloop.io(self.sock, pyev.EV_WRITE,
self.handle_connected)
self.sock_write_w.start()
return
elif error:
logger.error(
'Error while trying to connect to cc-server using %s',
sockaddr,
)
continue
else:
# connection already succeeded, very unlikely as we're using TCP
self._goto(self.handle_handshake)
return
def handle_connected(self, watcher=None, revents=None):
# check that there is no error
error = self.sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if error:
logger.error(
'Error while trying to connect to cc-server using %s,'
' (Errno %s: %s)', self.current_addr[-1],
error, os.strerror(error),
)
self._goto(self.handle_error)
return
self._goto(self.handle_handshake)
return
def cleanup_connect(self):
if self.sock_write_w is not None:
self.sock_write_w.stop()
del self.sock_write_w
if self.tick_w.active:
self.tick_w.stop()
del self.tick_w
def init_handshake(self):
self.sock = ssl.wrap_socket(self.sock, do_handshake_on_connect=False,
ssl_version=ssl.PROTOCOL_TLSv1)
self.sock_read_w = self.loop.evloop.io(self.sock, pyev.EV_READ,
self.handle_handshake)
self.sock_write_w = self.loop.evloop.io(self.sock, pyev.EV_WRITE,
self.handle_handshake)
def handle_handshake(self, watcher=None, revents=None):
assert self.sock is not None
try:
self.sock.do_handshake()
except ssl.SSLError as exc:
if exc.args[0] == ssl.SSL_ERROR_WANT_READ:
self.sock_write_w.stop()
self.sock_read_w.start()
elif exc.args[0] == ssl.SSL_ERROR_WANT_WRITE:
self.sock_read_w.stop()
self.sock_write_w.start()
else:
logger.exception('SSL error:')
self._goto(self.handle_error)
return
except socket.error as exc:
logger.error('Socket error while doing handshake, (errno %s: %s)',
exc.errno, exc.strerror)
self._goto(self.handle_error)
return
else:
# create rpc_connection object
self.rpc_con = RpcConnection(
sock=self.sock,
loop=self.loop.evloop,
handler=self.loop.rpc_handler,
on_disconnect=self.loop.restart_rpc_connection,
)
# since the socket is referenced by rpc_con, we don't need it
# anymore
self.sock = None
self._goto(self.handle_authentication)
return
def cleanup_handshake(self):
self.sock_read_w.stop()
del self.sock_read_w
self.sock_write_w.stop()
del self.sock_write_w
def init_authentication(self):
self.timeout_w = self.loop.evloop.timer(
self.AUTH_TIMEOUT, 0., self.handle_authentication_timeout)
# since the sjrpc connection start in fallback mode, it could block so
# we need to check this before attempting authentication;
# as there is no reliable way to check when we can authenticate, we try
# every 2 seconds
self.tick_w = self.loop.evloop.timer(
.5, 2., self.handle_authentication_tick)
self.auth_id = None
def handle_authentication(self):
assert self.sock is None
assert self.rpc_con is not None
self.tick_w.start()
def handle_authentication_tick(self, watcher=None, revents=None):
# check is fallback mode on sjrpc is set otherwise our call would block
# the loop
if not self.rpc_con._event_fallback.is_set():
logger.debug('Will try authentication again latter')
return
self.tick_w.stop()
self.timeout_w.start()
# try to authenticate
try:
self.auth_id = self.rpc_con.rpc.async_call_cb(
self.handle_authentication_cb,
'authentify',
self.loop.config.server_user,
self.loop.config.server_passwd,
)
except RpcError as exc:
if exc.exception == 'RpcConnectionError':
logger.error('Authentication failed: connection lost')
else:
logger.exception('Unexpected exception while authenticating')
self._goto(self.handle_error)
return
def handle_authentication_cb(self, call_id, response=None, error=None):
# RPC callback
assert call_id == self.auth_id
if error is not None:
# we got an error
logger.error('Error while authenticating with cc-server: %s("%s")',
error['exception'], error.get('message', ''))
self._goto(self.handle_error)
return
self.loop.rpc_con = self.rpc_con
# The rest is up to the subclass :)
self.handle_authentication_response(response)
def handle_authentication_response(self, response):
"""To be subclassed. Decides what to do about authentication response."""
raise NotImplementedError
def handle_authentication_timeout(self, watcher, revents):
logger.error('Timeout while authenticating with cc-server %s',
self.current_addr[-1])
self._goto(self.handle_error)
def cleanup_authentication(self):
if self.tick_w.active:
self.tick_w.stop()
del self.tick_w
if self.timeout_w.active:
self.timeout_w.stop()
del self.timeout_w
def handle_done(self):
logger.info('Successfully authenticated with role %s', str(self.loop.role))
self.stop()
def handle_error(self):
# cleanup and goto handle_connect
if self.rpc_con is not None:
self.rpc_con.rpc._on_disconnect = None # we don't want the cb
self.rpc_con.shutdown()
self.rpc_con = None
elif self.sock is not None:
self.sock.close()
self.sock = None
self._goto(self.handle_connect)
return
def start(self):
self._goto(self.handle_name)
return
def stop(self):
self._goto(None)
return
class MainLoop(object):
# DEFAULTS_TAGS = tuple()
# CONNECT_CLASS = RPCStartHandler
# CONFIG_CLASS = None
def __init__(self, config_path):
self.config_path = config_path
# load config variables
try:
self.load_config()
except ConfigError as exc:
sys.exit(exc.message)
# configure logging
self.configure_logging()
# always run evloop as debug, we don't want exceptions to be mute
self.evloop = pyev.default_loop(flags=pyev.EVFLAG_SIGNALFD,
debug=True)
# keeps track of libev loop thread, create threading queue and an async
# watcher for performing work in libev thread from external thread
# see decorators in utils.py
self.pyev_thread = threading.current_thread()
self.async_queue = Queue.Queue()
self.async_watcher = self.evloop.async(self.async_work_cb)
# set signal watchers
self.signals = {
signal.SIGINT: self.stop,
signal.SIGTERM: self.stop,
signal.SIGUSR1: self.reload,
}
# turn into real watchers
self.signals = dict((
signal_,
self.evloop.signal(signal_, cb),
) for signal_, cb in self.signals.iteritems())
# rpc connection
self.rpc_con = None
self.connect = self.CONNECT_CLASS(self)
self.reconnect = None
# role
self.role = None
self.main_plugin = None
# tag database
self.tag_db = RootTagDB(self, tags=self.DEFAULT_TAGS)
# handlers
self.rpc_handler = dict(
get_tags=partial(threadless(get_tags), self.tag_db['__main__']),
sub_tags=self.sub_tags,
)
# plugins
self.registered_plugins = set()
@property
def rpc_connected(self):
return self.rpc_con is not None
@property
def rpc_authenticated(self):
return self.rpc_connected and self.role is not None
def load_config(self):
self.config = self.CONFIG_CLASS(self.config_path)
def configure_logging(self):
raise NotImplementedError
def call_in_main_thread(self, func, *args, **kwargs):
"""Performs work in pyev thread.
"""
if threading.current_thread() is self.pyev_thread:
# if called from main thread, don't do anything special
return func(*args, **kwargs)
else:
class Return(object):
def __init__(self):
self.return_value = None
self.exc = None
return_ = Return()
event = threading.Event()
def cb():
try:
return_.return_value = func(*args, **kwargs)
except Exception as e:
return_.exc = e
finally:
event.set()
self.async_queue.put(cb)
self.async_watcher.send()
# wait for work to be processed and raise error
event.wait()
if return_.exc is not None:
raise return_.exc
return return_.return_value
def async_work_cb(self, watcher, revents):
logger.debug('Async work processing')
while True:
try:
work = self.async_queue.get_nowait()
except Queue.Empty:
break
try:
work()
except Exception:
# this should not happen but just in case
logger.exception('Error async work')
# RPC handlers definitions
@threadless
def sub_tags(self, sub_id, tags=None):
if sub_id == '__main__':
# FIXME should we raise ?
logger.debug('Invalid request for sub object')
return {}
sub_db = self.tag_db.get(sub_id)
if sub_db is None:
# FIXME should we also raise here ?
logger.debug('Failed to find sub_id %s', sub_id)
return {}
return get_tags(sub_db, tags)
@threadless
def job_list(self):
pass
# End RPC handlers definitions
def reset_handler(self, name, handl):
self.rpc_handler[name] = handl
def remove_handler(self, name):
self.rpc_handler.pop(name, None)
def register_plugin(self, plugin):
# keep track of registered plugins
if plugin in self.registered_plugins:
raise PluginError('Plugin was already registered')
self.registered_plugins.add(plugin)
# register tags
plugin.tag_db.set_parent(self.tag_db)
# register handlers
for k, v in plugin.rpc_handler.iteritems():
self.reset_handler(k, v)
plugin.start()
def unregister_plugin(self, plugin):
try:
self.registered_plugins.remove(plugin)
except KeyError:
raise PluginError('Plugin was not registered, cannot remove')
# remove tags
plugin.tag_db.set_parent(None)
# remove handlers
for handler_name in plugin.rpc_handler:
self.remove_handler(handler_name)
plugin.stop()
def close_plugins(self):
"""Unregister all plugins from the loop."""
for plugin in self.registered_plugins.copy():
self.unregister_plugin(plugin)
def restart_rpc_connection(self, *args):
if not self.rpc_connected:
return
# clear connection
self.rpc_con = None
logger.error('Lost connection to the cc-server, will attempt'
' reconnection')
# reconnection atempt in one second
self.reconnect = self.evloop.timer(
1., 0., self.restart_rpc_connection_cb)
self.reconnect.start()
def restart_rpc_connection_cb(self, *args):
# attempt to connect to the cc-server again
self.connect.start()
self.reconnect.stop()
self.reconnect = None
def start(self):
logger.info('Starting node')
self.async_watcher.start()
for signal_ in self.signals.itervalues():
signal_.start()
logger.debug('About to connect')
self.connect.start()
logger.debug('About to start ev_loop')
try:
self.evloop.start()
except:
logger.exception('Unexpected exception while running main loop')
sys.exit(1)
def stop(self, watcher=None, revents=None):
logger.info('Exiting node...')
if self.connect is not None:
self.connect.stop()
if self.reconnect is not None:
self.reconnect.stop()
# close rpc
if self.rpc_con is not None:
# disable callback to prevent trampoline calls
self.rpc_con.rpc._on_disconnect = None
self.rpc_con.shutdown()
# close all plugins
for plugin in self.registered_plugins:
plugin.stop()
self.registered_plugins = set()
# FIXME check for closing of main tags that were not in plugin
self.role = None
self.main_plugin = None
self.evloop.stop()
def reload(self, watcher=None, revents=None):
logger.info('Reloading logging configuration...')
try:
self.load_config()
self.configure_logging()
except Exception:
logger.exception('Invalid config file')
cc-common-v8/cloudcontrol/common/client/plugins.py 0000664 0000000 0000000 00000007631 12600735274 0022611 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
"""Plugins helpers for Cloud Control clients."""
from inspect import getmembers
from cloudcontrol.common.client.tags import TagDB
def rpc_handler_decorator_factory(attr_name):
"""Can be used to create custom decorators for marking callable objects as
RPC handlers
"""
def rpc_handler(name_or_func=None):
"""RPC handler decorator
Declares a callable as an RPC handler
:param name_or_func: allow decorator to be used in two manners:
# the first case is to export the handler using the same name as the
# function
@rpc_handler
def plop():
pass
# the second case allow to specify the RPC handler exported name
@rpc_handler('plop')
def plop_handler():
pass
"""
def decorator(callable_):
"""Decorator for marking a callable as an RPC handler."""
assert callable(callable_)
setattr(callable_, attr_name, name)
return callable_
if callable(name_or_func):
# we are in the first case
name = True
return decorator(name_or_func)
# else, second case
name = name_or_func
return decorator
return rpc_handler
def get_rpc_handlers(o, marker):
"""Get RPC handlers members of an object
:param o: the object to instrospect
:param str marker: the attribute to check to decide whether the member is an
RPC handler
"""
def get_marker(member):
return getattr(member, marker, None)
return dict((
# return handler name to be the callable object name or the one defined
# through the decorator
get_marker(v) if isinstance(get_marker(v), basestring) else k,
v,
) for k, v in getmembers(
o, lambda m: callable(m) and get_marker(m) is not None))
#: basic decorator to declare RPC handlers
rpc_handler_marker = '_rpc_handler'
rpc_handler = rpc_handler_decorator_factory(rpc_handler_marker)
class Base(object):
"""Example skeleton plugin for clients.
If you want to create your own plugin, you may create an object that would
quack just like this one or just inherit from this class.
"""
def __init__(self, *args, **kwargs):
"""
:param loop: MainLoop instance
"""
#: MainLoop instance
self.main = kwargs.pop('loop')
# plugins may define tags (see :mod:`ccnode.tags`)
self.tag_db = TagDB()
# plugins may define handler functions that would be called by the
# server
self.rpc_handler = get_rpc_handlers(self, rpc_handler_marker)
# tag_db and rpc_handler can be implemented as properties if more logic
# is needed
def __hash__(self):
"""This method is used when registering a plugin in the main loop.
By default, only one instance is allowed. Subclasses can overide this
method to change this behaviour.
"""
return hash(self.__class__.__name__)
def start(self):
"""Used to start pyev watchers."""
pass
def stop(self):
"""Cleanup for plugins, can be used to clean pyev watchers."""
self.main = None
# TODO dependencies
cc-common-v8/cloudcontrol/common/client/tags.py 0000664 0000000 0000000 00000056255 12600735274 0022074 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
import inspect
import logging
import weakref
import threading
from functools import partial, update_wrapper
from itertools import chain
from collections import defaultdict
from cloudcontrol.common.client.utils import main_thread
from cloudcontrol.common.client.exc import TagConflict
logger = logging.getLogger(__name__)
def ensure_unicode(text, encoding='utf-8'):
""" Ensure input string is unicode, or decode it.
"""
if isinstance(text, unicode):
return text
elif isinstance(text, (str, buffer)):
return unicode(text, encoding)
else:
return unicode(text)
class Tag(object):
"""``class`` that abstract tags and act as a simple container."""
def __init__(self, name, valuable, ttl=-1, refresh=None, parent=None,
background=False):
"""
:param string name: tag name
:param valuable: something that gives tag value, string or callable
:param None,int ttl: Time to live for caching the tags, possible values
are:
* None (means no ttl)
* -1 (means infinite ttl)
* positive integer in seconds
:param None,int refresh: period used to refresh tag value on the node
:param object parent: parent object the tag is attached to, it may be
needed for the tag callable to process the tag
:param bool background: calculate the tag value in a background thread
(migth be usefull to not block the event loop)
"""
self.name = name
self.value = None
self.is_function = False
self.ttl = ttl
self.refresh = refresh
self.parent = parent if parent is None else weakref.proxy(parent)
self.background = bool(background)
if inspect.isfunction(valuable):
self.is_function = True
args_count = len(inspect.getargspec(valuable).args)
if args_count > 1:
raise TypeError('Tag function take at most 1 argument')
elif args_count == 1:
self._calculate_value = partial(valuable, self.parent)
else:
self._calculate_value = valuable
elif callable(valuable): # Handle non-function callables
self.is_function = True
self._calculate_value = valuable
else:
self.value = valuable
#: timer instance
self.watcher = None
#: async watcher in case of background is True
self.async = None
# special arguments for tag db
self.db = None
self.sub_id = None
def calculate_value(self):
try:
self.value = self._calculate_value()
except Exception:
logger.exception('Cannot calculate tag value for %s', self.name)
self.value = None
# logger.debug('Calculate Tag(%s) = %s', self.name, self.value)
def update_value(self):
"""Called when the tag value may change."""
prev_value = self.value
if not self.background:
self.calculate_value()
if self.value != prev_value:
self._handle_registration(prev_value)
return
# else
# create a thread that will run the calculate_value method
def thread_run():
self.calculate_value()
self.async.send()
# keep previous value in watcher's opaque
self.async.data = prev_value
self.async.start()
th = threading.Thread(target=thread_run)
th.daemon = True
th.start()
def async_cb(self, watcher, revents):
if watcher.data != self.value:
self._handle_registration(watcher.data)
watcher.data = None
watcher.stop()
def _handle_registration(self, prev_value):
if self.db is None:
# when we calculate the tag, the latter could raise an exception
# and could provoke a deconnection to the libvirt for example,
# that will also provoke a deregister of some tags which might
# include the current (in case this happens, just return)
return
if prev_value is None:
# we need to register tag again
if self.sub_id == '__main__':
logger.debug('Register tag %s', self.name)
self.db.rpc_register_tag(self)
else:
logger.debug('Register sub tag %s.%s', self.sub_id,
self.name)
self.db.rpc_register_sub_tag(self.sub_id, self)
elif self.value is None:
# we drop the tag
if self.sub_id == '__main__':
logger.debug('Unregister tag %s', self.name)
self.db.rpc_unregister_tag(self.name)
else:
logger.debug(
'Unregister sub tag %s.%s', self.sub_id, self.name)
self.db.rpc_unregister_sub_tag(self.sub_id, self.name)
# if tag is not pushed
elif self.ttl != -1:
return
else:
# update the tag value
logger.debug('Update tag value %s', self.name)
self.db.rpc_update_tag(self.sub_id, self)
def start(self, loop):
"""
:param loop: pyev loop
"""
if self.background:
self.async = loop.async(self.async_cb)
if not self.is_function:
return
if self.refresh is None and not self.background:
self.calculate_value()
return
# TODO more sophisticated calculation with event propagation
refresh = .0 if self.background else float(self.refresh)
self.watcher = loop.timer(.0, refresh, lambda *args: self.update_value())
self.watcher.start()
def stop(self):
if self.watcher is not None:
self.watcher.stop()
self.watcher = None
def tag_inspector(mod, parent=None):
"""Inspect module to find tags.
:param module mod: module to inspect
Currently there are two ways to define a tag inside a module:
* affect a string to a variable, the variable name will be the tag
name
* define a function that returns a value as a string or None (meaning
the tag doesn't exist on the host), as you guessed the function name
will define the tag name
"""
tags = []
for n, m in inspect.getmembers(mod): # (name, member)
# only keep strings or functions as tags
if getattr(m, '__module__', None) != mod.__name__ or (
n.startswith('_')):
continue
elif isinstance(m, (str, unicode)):
# if string, it means it is constant, then ttl = -1
ttl = -1
refresh = None
background = False
elif inspect.isfunction(m):
# if function take function ttl argument or set -1 by default
ttl = getattr(m, 'ttl', -1)
refresh = getattr(m, 'refresh', None)
background = getattr(m, 'background', False)
else:
# whatever it is we don't care...
continue
logger.debug('Introspected %s with ttl %s, refresh %s, background %s for object %s',
n, ttl, refresh, background, parent)
# finally add the tag
tags.append(Tag(n, m, ttl, refresh, parent, background))
return tags
# decorators for tag inspector
def ttl(value):
def decorator(func):
func.ttl = value
return func
return decorator
def refresh(value):
def decorator(func):
func.refresh = value
return func
return decorator
def background(max_concurrent=None):
def background_decorator(func):
func.background = True
semaphore = threading.Semaphore(max_concurrent) if max_concurrent else None
if semaphore is not None:
if not len(inspect.getargspec(func).args):
def wrapper():
with semaphore:
return func()
else:
def wrapper(dom):
with semaphore:
return func(dom)
wrapper = update_wrapper(wrapper, func)
return wrapper
else:
return func
return background_decorator
def get_tags(tags_dict, tags=None):
"""Helper to get tags.
:param tags_dict: dict containing :class:`Tag` objects
:param tags: list of tags to get (None mean all tags)
"""
# logger.debug('Tags request: %s', unicode(tags))
if tags is None:
tags = tags_dict.iterkeys()
else:
tags = set(tags) & set(tags_dict)
result = dict((
t, # tag name
tags_dict[t].value,
) for t in tags)
# logger.debug('Returning: %s', unicode(result))
return result
def rpc_simple_cb(log_msg):
"""Log a message in case of error of the rpc call.
It is a factory that is used in TagDB
:param str log_msg: format logging message with 3 %s arguments
(opaque, error class, error message)
"""
def cb(self, call_id, response, error):
opaque = self.async_calls.pop(call_id)
if error:
logger.error(log_msg, opaque, error['exception'],
error['message'])
return
logger.debug('Simple cb for %s succeed', opaque)
return cb
class TagDB(object):
"""Tag database. FIXME comment
Handles common operations such as registering tag on the cc-server,
updating its values, etc.
TagDB can have a parent TagDB, in this case, the latter could handle tag
registration on the cc-server.
"""
def __init__(self, parent_db=None, tags=None):
"""
:param TagDB parent_db: TagDB parent object
:param iterable tags: initial tags
"""
self._parent = None
if tags is None:
tags = tuple()
self.db = defaultdict(
dict,
__main__=dict(), # tags for main object
# others objects
)
#: associate type for each sub object
self._object_types = dict()
for tag in tags:
self.add_tag(tag)
self.set_parent(parent_db)
def set_parent(self, parent):
"""Set parent tag database."""
# check if previous parent
if self._parent is not None:
# we must remove tags from old parent
self._parent.remove_tags(self.db['__main__'])
for sub_id in self.db:
if sub_id == '__main__':
continue
self._parent.remove_sub_object(sub_id)
# set new parent
self._parent = parent
if self._parent is not None:
# add tags in new parent
self._parent.add_tags(self.db['__main__'].itervalues())
for sub_id, db in self.db.iteritems():
if sub_id == '__main__':
continue
self._parent.add_sub_object(sub_id, db.itervalues(),
self._object_types[sub_id])
def check_tags_conflict(self, *tag_names):
"""Checks a list of tag names that might conflict before inserting in
TagDB hierarchy
.. warning::
This is in no way a guarantee that following inserts will succeed.
"""
conflicts = []
parent_check = []
for name in tag_names:
if name in self.db['__main__']:
conflicts.append(name)
else:
parent_check.append(name)
if self._parent is not None and parent_check:
conflicts.extend(self._parent.check_tags_conflict(*parent_check))
return conflicts
# tag handling part, used by plugins
def add_tags(self, tags):
"""
:param iterable tags: list of tags to add
"""
for tag in tags:
self.add_tag(tag)
def add_sub_tags(self, sub_id, tags):
for tag in tags:
self.add_sub_tag(sub_id, tag)
def remove_tags(self, tag_names):
"""
:param iterable tag_names: list of tag names to remove
"""
for name in tag_names:
self.remove_tag(name)
def check_tag(self, tag):
"""Checks that a given tag object is the same instance in the db
Usefull for checking before removing.
"""
return id(self.db['__main__'].get(tag.name, None)) == id(tag)
def add_tag(self, tag):
if tag.name in self.db['__main__']:
raise TagConflict(
'A tag with the name %s is already registered' % tag.name)
# first add the tag to the child db, in case of conflict in the parent,
# the tag is recorded in the child thus if the latter's parent is
# changed it can be recorded again after
self.db['__main__'][tag.name] = tag
# set special attributes on tag instance
if self._parent is not None:
self._parent.add_tag(tag)
def remove_tag(self, tag_name):
tag = self.db['__main__'].pop(tag_name)
if self._parent is not None and self._parent.check_tag(tag):
# remove tag in parent db only if it is the same instance
self._parent.remove_tag(tag_name)
def check_sub_tag(self, sub_id, tag):
return id(self.db[sub_id].get(tag.name, None)) == id(tag)
def add_sub_tag(self, sub_id, tag):
if tag.name in self.db[sub_id]:
raise TagConflict(
'A tag with the name %s is already registered' % tag.name)
self.db[sub_id][tag.name] = tag
if self._parent is not None:
self._parent.add_sub_tag(sub_id, tag)
def remove_sub_tag(self, sub_id, tag_name):
tag = self.db[sub_id].pop(tag_name)
if self._parent is not None and self._parent.check_tag(tag):
self._parent.remove_sub_tag(sub_id, tag_name)
def add_sub_object(self, sub_id, tags, type_):
self._object_types[sub_id] = type_
if self._parent is not None:
# tags will be added after
self._parent.add_sub_object(sub_id, tuple(), type_)
# add sub object tags
for t in tags:
self.add_sub_tag(sub_id, t)
def remove_sub_object(self, sub_id):
del self.db[sub_id]
del self._object_types[sub_id]
if self._parent is not None:
self._parent.remove_sub_object(sub_id)
# dict like
def get(self, key, default=None):
return self.db.get(key, default)
def __getitem__(self, key):
return self.db[key]
def keys(self):
return self.db.keys()
def iteritems(self):
return self.db.iteritems()
def itervalues(self):
return self.db.itervalues()
# end dict like
class ParentWrapper(object):
def __init__(self, sub_id, type_, parent):
self.sub_id = sub_id
self._parent = parent
self._parent.add_sub_object(self.sub_id, tuple(), type_)
def __del__(self):
self._parent.remove_sub_object(self.sub_id)
def _not_implemented(self, *args, **kwargs):
raise NotImplementedError
def add_tags(self, tags):
for t in tags:
self.add_tag(t)
def remove_tags(self, tag_names):
for n in tag_names:
self.remove_tag(n)
def add_tag(self, tag):
self._parent.add_sub_tag(self.sub_id, tag)
def remove_tag(self, tag_name):
self._parent.remove_sub_tag(self.sub_id, tag_name)
def __getattr__(self, name):
if name in ('add_sub_tag', 'remove_sub_tag', 'add_sub_object',
'remove_sub_object'):
return self._not_implemented
return getattr(self._parent, name)
class RootTagDB(TagDB):
"""Root tag database.
It takes care of tag registration with cc-server. It has no parent.
"""
def __init__(self, main, tags=None):
"""
:param main: MainLoop instance
:param tags: initial tags
"""
self.main = main
#: dict for RPC async call storage, keep a part of log message
self.async_calls = dict()
TagDB.__init__(self, tags=tags)
def set_parent(self, parent):
if parent is not None:
raise TypeError('Cannot set parent on RootTagDB')
# RPC part
def rpc_call(self, opaque, callback, remote_name, *args, **kwargs):
"""Local helper for all rpc calls.
:param opaque: data to associate with the async call
:param callable callback: callback when call is done, signature is the
same as for :py:meth:`RpcConnection.async_call_cb`
:param remote_name: remote method name
:param \*args: arguments for the call
:param \*\*kwargs: keyword arguments for the call
"""
# call only if connected and authenticated to the cc-server
if self.main.rpc_authenticated:
# logger.debug('RPC call %s %s', remote_name, args)
self.async_calls[self.main.rpc_con.rpc.async_call_cb(
callback, remote_name, *args, **kwargs)] = opaque
def rpc_register(self):
"""Register all objects and tags on the cc-server.
This is used just after a (re)connection to the cc-server is made.
"""
for sub_id in self.db:
if sub_id == '__main__':
continue
self.rpc_register_sub_object(sub_id, self._object_types[sub_id])
# register tags on the cc-server
for tag in self.db['__main__'].itervalues():
if tag.value is not None:
self.rpc_register_tag(tag)
# register sub tags on the cc-server
for tag in chain.from_iterable(
db.itervalues() for name, db in self.db.iteritems()
if name != '__main__'):
if tag.value is not None:
self.rpc_register_sub_tag(tag.sub_id, tag)
def rpc_register_sub_object(self, sub_id, type_):
# register object on the cc-server
self.rpc_call(sub_id, self.rpc_object_register_cb,
'register', sub_id, type_)
def rpc_unregister_sub_object(self, sub_id):
self.rpc_call(
sub_id, self.rpc_object_register_cb, 'unregister', sub_id)
#: this is a method
rpc_object_register_cb = rpc_simple_cb(
'Error while trying to register the object %s, %s("%s")')
rpc_object_unregister_cb = rpc_simple_cb(
'Error while trying to unregister the object %s, %s("%s")')
def rpc_register_tag(self, tag):
logger.debug('RPC register tag %s', tag.name)
ttl = None if tag.ttl == -1 else tag.ttl
self.rpc_call(tag.name, self.rpc_tag_register_cb, 'tags_register',
tag.name, ttl, ensure_unicode(tag.value))
def rpc_unregister_tag(self, tag_name):
logger.debug('RPC unregister tag %s', tag_name)
self.rpc_call(tag_name, self.rpc_tag_unregister_cb, 'tags_unregister',
tag_name)
def rpc_update_tag(self, sub_id, tag):
"""Update tag value on cc-server."""
logger.debug('RPC update tag %s(%s)', tag.name, sub_id)
if sub_id == '__main__':
self.rpc_call(sub_id + tag.name, self.rpc_update_tag_cb,
'tags_update', tag.name, ensure_unicode(tag.value))
else:
self.rpc_call(sub_id + tag.name, self.rpc_update_tag_cb,
'sub_tags_update', sub_id, tag.name, ensure_unicode(tag.value))
def rpc_register_sub_tag(self, sub_id, tag):
logger.debug('RPC register tag %s(%s)', tag.name, sub_id)
ttl = None if tag.ttl == -1 else tag.ttl
self.rpc_call(tag.name, self.rpc_sub_tag_register_cb,
'sub_tags_register', sub_id, tag.name, ttl,
ensure_unicode(tag.value))
def rpc_unregister_sub_tag(self, sub_id, tag_name):
logger.debug('RPC unregister tag %s(%s)', tag_name, sub_id)
self.rpc_call(tag_name, self.rpc_sub_tag_unregister_cb,
'sub_tags_unregister', sub_id, tag_name)
#: this is a method
rpc_tag_register_cb = rpc_simple_cb(
'Error while trying to register tag %s, %s("%s")')
rpc_tag_unregister_cb = rpc_simple_cb(
'Error while trying to unregister tag %s, %s("%s")')
rpc_sub_tag_register_cb = rpc_simple_cb(
'Error while registering sub tag %s, %s("%s")')
rpc_sub_tag_unregister_cb = rpc_simple_cb(
'Error while unregistering sub tag %s, %s("%s")')
rpc_update_tag_cb = rpc_simple_cb(
'Error while trying to update tag %s, %s("%s")')
# end RPC part
@main_thread
def add_tag(self, tag):
if tag.name in self.db['__main__']:
raise TagConflict(
'A tag with the name %s was already registered' % tag.name)
# set special attributes on tag instance
tag.db = self
tag.sub_id = '__main__'
tag.start(self.main.evloop)
# register tag on the cc-server
if tag.value is not None:
self.rpc_register_tag(tag)
self.db['__main__'][tag.name] = tag
@main_thread
def remove_tag(self, tag_name):
tag = self.db['__main__'].pop(tag_name)
tag.db = None
tag.sub_id = None
tag.stop()
# unregister tag on the cc-server
if tag.value is not None:
self.rpc_unregister_tag(tag_name)
@main_thread
def add_sub_tag(self, sub_id, tag):
if tag.name in self.db[sub_id]:
raise TagConflict(
'A tag with the name %s (%s) was already registered' %
(tag.name, sub_id))
tag.db = self
tag.sub_id = sub_id
tag.start(self.main.evloop)
# register tag to the cc-server
if tag.value is not None:
self.rpc_register_sub_tag(sub_id, tag)
self.db[sub_id][tag.name] = tag
@main_thread
def remove_sub_tag(self, sub_id, tag_name):
tag = self.db[sub_id].pop(tag_name)
tag.db = None
tag.sub_id = None
tag.stop()
# unregister tag to the cc-server
if tag.value is not None:
self.rpc_unregister_sub_tag(sub_id, tag_name)
@main_thread
def add_sub_object(self, sub_id, tags, type_):
self.rpc_register_sub_object(sub_id, type_)
self._object_types[sub_id] = type_
# add sub object tags
for t in tags:
self.add_sub_tag(sub_id, t)
@main_thread
def remove_sub_object(self, sub_id):
for tag in self.db[sub_id].itervalues():
tag.stop()
tag.db = None
tag.sub_id = None
# we don't need to unregister each sub tag on the cc-server because
# it will be done when we unregister the object
del self.db[sub_id]
del self._object_types[sub_id]
self.rpc_unregister_sub_object(sub_id)
cc-common-v8/cloudcontrol/common/client/utils.py 0000664 0000000 0000000 00000002074 12600735274 0022264 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
from functools import wraps
def main_thread(func):
"""Decorator for plugin methods that need to be executed in pyev thread.
Actually anything that has an attribute `main` that points to an instance of
MainLoop.
"""
@wraps(func)
def decorated(self, *args, **kwargs):
return self.main.call_in_main_thread(func, self, *args, **kwargs)
return decorated
cc-common-v8/cloudcontrol/common/datastructures/ 0000775 0000000 0000000 00000000000 12600735274 0022346 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/datastructures/__init__.py 0000664 0000000 0000000 00000001266 12600735274 0024464 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
cc-common-v8/cloudcontrol/common/datastructures/ordereddict.py 0000664 0000000 0000000 00000020156 12600735274 0025214 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Ordered dict implementation.
This code is taken from the standard library of Python 2.7.
"""
from collections import MutableMapping, KeysView, ValuesView, ItemsView
try:
from thread import get_ident as _get_ident
except ImportError:
from dummy_thread import get_ident as _get_ident
################################################################################
### OrderedDict
################################################################################
class OrderedDict(dict):
'Dictionary that remembers insertion order'
# An inherited dict maps keys to values.
# The inherited dict provides __getitem__, __len__, __contains__, and get.
# The remaining methods are order-aware.
# Big-O running times for all methods are the same as regular dictionaries.
# The internal self.__map dict maps keys to links in a doubly linked list.
# The circular doubly linked list starts and ends with a sentinel element.
# The sentinel element never gets deleted (this simplifies the algorithm).
# Each link is stored as a list of length three: [PREV, NEXT, KEY].
def __init__(self, *args, **kwds):
'''Initialize an ordered dictionary. The signature is the same as
regular dictionaries, but keyword arguments are not recommended because
their insertion order is arbitrary.
'''
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
try:
self.__root
except AttributeError:
self.__root = root = [] # sentinel node
root[:] = [root, root, None]
self.__map = {}
self.__update(*args, **kwds)
def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__):
'od.__setitem__(i, y) <==> od[i]=y'
# Setting a new item creates a new link at the end of the linked list,
# and the inherited dictionary is updated with the new key/value pair.
if key not in self:
root = self.__root
last = root[PREV]
last[NEXT] = root[PREV] = self.__map[key] = [last, root, key]
dict_setitem(self, key, value)
def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__):
'od.__delitem__(y) <==> del od[y]'
# Deleting an existing item uses self.__map to find the link which gets
# removed by updating the links in the predecessor and successor nodes.
dict_delitem(self, key)
link_prev, link_next, key = self.__map.pop(key)
link_prev[NEXT] = link_next
link_next[PREV] = link_prev
def __iter__(self):
'od.__iter__() <==> iter(od)'
# Traverse the linked list in order.
NEXT, KEY = 1, 2
root = self.__root
curr = root[NEXT]
while curr is not root:
yield curr[KEY]
curr = curr[NEXT]
def __reversed__(self):
'od.__reversed__() <==> reversed(od)'
# Traverse the linked list in reverse order.
PREV, KEY = 0, 2
root = self.__root
curr = root[PREV]
while curr is not root:
yield curr[KEY]
curr = curr[PREV]
def clear(self):
'od.clear() -> None. Remove all items from od.'
for node in self.__map.itervalues():
del node[:]
root = self.__root
root[:] = [root, root, None]
self.__map.clear()
dict.clear(self)
# -- the following methods do not depend on the internal structure --
def keys(self):
'od.keys() -> list of keys in od'
return list(self)
def values(self):
'od.values() -> list of values in od'
return [self[key] for key in self]
def items(self):
'od.items() -> list of (key, value) pairs in od'
return [(key, self[key]) for key in self]
def iterkeys(self):
'od.iterkeys() -> an iterator over the keys in od'
return iter(self)
def itervalues(self):
'od.itervalues -> an iterator over the values in od'
for k in self:
yield self[k]
def iteritems(self):
'od.iteritems -> an iterator over the (key, value) pairs in od'
for k in self:
yield (k, self[k])
update = MutableMapping.update
__update = update # let subclasses override update without breaking __init__
__marker = object()
def pop(self, key, default=__marker):
'''od.pop(k[,d]) -> v, remove specified key and return the corresponding
value. If key is not found, d is returned if given, otherwise KeyError
is raised.
'''
if key in self:
result = self[key]
del self[key]
return result
if default is self.__marker:
raise KeyError(key)
return default
def setdefault(self, key, default=None):
'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
if key in self:
return self[key]
self[key] = default
return default
def popitem(self, last=True):
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if false.
'''
if not self:
raise KeyError('dictionary is empty')
key = next(reversed(self) if last else iter(self))
value = self.pop(key)
return key, value
def __repr__(self, _repr_running={}):
'od.__repr__() <==> repr(od)'
call_key = id(self), _get_ident()
if call_key in _repr_running:
return '...'
_repr_running[call_key] = 1
try:
if not self:
return '%s()' % (self.__class__.__name__,)
return '%s(%r)' % (self.__class__.__name__, self.items())
finally:
del _repr_running[call_key]
def __reduce__(self):
'Return state information for pickling'
items = [[k, self[k]] for k in self]
inst_dict = vars(self).copy()
for k in vars(OrderedDict()):
inst_dict.pop(k, None)
if inst_dict:
return (self.__class__, (items,), inst_dict)
return self.__class__, (items,)
def copy(self):
'od.copy() -> a shallow copy of od'
return self.__class__(self)
@classmethod
def fromkeys(cls, iterable, value=None):
'''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
If not specified, the value defaults to None.
'''
self = cls()
for key in iterable:
self[key] = value
return self
def __eq__(self, other):
'''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
while comparison to a regular mapping is order-insensitive.
'''
if isinstance(other, OrderedDict):
return len(self)==len(other) and self.items() == other.items()
return dict.__eq__(self, other)
def __ne__(self, other):
'od.__ne__(y) <==> od!=y'
return not self == other
# -- the following methods support python 3.x style dictionary views --
def viewkeys(self):
"od.viewkeys() -> a set-like object providing a view on od's keys"
return KeysView(self)
def viewvalues(self):
"od.viewvalues() -> an object providing a view on od's values"
return ValuesView(self)
def viewitems(self):
"od.viewitems() -> a set-like object providing a view on od's items"
return ItemsView(self)
cc-common-v8/cloudcontrol/common/datastructures/orderedset.py 0000664 0000000 0000000 00000005704 12600735274 0025066 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" OrderedSet Python implementation.
This snippet of code is taken from http://code.activestate.com/recipes/576694/
Written by Raymond Hettinger's and licenced under the MIT Licence.
Comments::
Runs on Py2.6 or later (and runs on 3.0 or later without any modifications).
Implementation based on a doubly linked link and an internal dictionary.
This design gives OrderedSet the same big-Oh running times as regular sets
including O(1) adds, removes, and lookups as well as O(n) iteration.
"""
import collections
KEY, PREV, NEXT = range(3)
class OrderedSet(collections.MutableSet):
def __init__(self, iterable=None):
self.end = end = []
end += [None, end, end] # sentinel node for doubly linked list
self.map = {} # key --> [key, prev, next]
if iterable is not None:
self |= iterable
def __len__(self):
return len(self.map)
def __contains__(self, key):
return key in self.map
def add(self, key):
if key not in self.map:
end = self.end
curr = end[PREV]
curr[NEXT] = end[PREV] = self.map[key] = [key, curr, end]
def discard(self, key):
if key in self.map:
key, prev, next = self.map.pop(key)
prev[NEXT] = next
next[PREV] = prev
def __iter__(self):
end = self.end
curr = end[NEXT]
while curr is not end:
yield curr[KEY]
curr = curr[NEXT]
def __reversed__(self):
end = self.end
curr = end[PREV]
while curr is not end:
yield curr[KEY]
curr = curr[PREV]
def pop(self, last=True):
if not self:
raise KeyError('set is empty')
key = next(reversed(self)) if last else next(iter(self))
self.discard(key)
return key
def __repr__(self):
if not self:
return '%s()' % (self.__class__.__name__,)
return '%s(%r)' % (self.__class__.__name__, list(self))
def __eq__(self, other):
if isinstance(other, OrderedSet):
return len(self) == len(other) and list(self) == list(other)
return set(self) == set(other)
def __del__(self):
self.clear() # remove circular references
cc-common-v8/cloudcontrol/common/helpers/ 0000775 0000000 0000000 00000000000 12600735274 0020733 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/helpers/__init__.py 0000664 0000000 0000000 00000001266 12600735274 0023051 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
cc-common-v8/cloudcontrol/common/helpers/formatter.py 0000664 0000000 0000000 00000002501 12600735274 0023306 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
import logging
class EncodingFormatter(logging.Formatter, object):
def __init__(self, fmt=None, datefmt=None, encoding='utf-8'):
super(EncodingFormatter, self).__init__(fmt, datefmt)
self._encoding = encoding
def formatException(self, ei):
expt = super(EncodingFormatter, self).formatException(ei)
if isinstance(expt, str):
expt = expt.decode(self._encoding, 'replace')
return expt
def format(self, record):
msg = super(EncodingFormatter, self).format(record)
if isinstance(msg, unicode):
msg = msg.encode(self._encoding, 'replace')
return msg
cc-common-v8/cloudcontrol/common/helpers/logger.py 0000664 0000000 0000000 00000002610 12600735274 0022563 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Logging helpers.
"""
import types
# This function is taken from logging package from Python stdlib v2.7.
def getChild(self, suffix):
"""
Get a logger which is a descendant to this one.
This is a convenience method, such that
logging.getLogger('abc').getChild('def.ghi')
is the same as
logging.getLogger('abc.def.ghi')
It's useful, for example, when the parent logger is named using
__name__ rather than a literal string.
"""
if self.root is not self:
suffix = '.'.join((self.name, suffix))
return self.manager.getLogger(suffix)
def patch_logging():
import logging
logging.Logger.getChild = types.UnboundMethodType(getChild, None, logging.Logger)
cc-common-v8/cloudcontrol/common/jobs/ 0000775 0000000 0000000 00000000000 12600735274 0020226 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/jobs/__init__.py 0000664 0000000 0000000 00000002231 12600735274 0022335 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Jobs manager of CloudControl.
The jobs manager is a component running on both server and nodes. It is in
charge of reporting of jobs status.
"""
from cloudcontrol.common.jobs.interface import JobsManagerInterface
from cloudcontrol.common.jobs.store import JobsStore
from cloudcontrol.common.jobs.job import Job, JobCancelError
from cloudcontrol.common.jobs.manager import JobsManager
__all__ = ['JobsManager', 'JobsManagerInterface', 'Job', 'JobsStore', 'JobCancelError']
cc-common-v8/cloudcontrol/common/jobs/helpers.py 0000664 0000000 0000000 00000002301 12600735274 0022236 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Bunch of helpers for jobs system.
"""
from threading import Lock
class AutoIncrementCounter(object):
""" Manage an auto incremented counter.
"""
def __init__(self, store):
self._store = store
self._current = self._store.get_id_counter()
self._lock = Lock()
def get(self):
""" Get a new incremented number.
"""
with self._lock:
self._current += 1
self._store.store_id_counter(self._current)
return self._current
cc-common-v8/cloudcontrol/common/jobs/interface.py 0000664 0000000 0000000 00000002321 12600735274 0022536 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Interface used to get events from the jobs manager.
"""
from abc import ABCMeta
class JobsManagerInterface(object):
__metaclass__ = ABCMeta
def __init__(self):
pass
def on_job_created(self, job):
""" Method called by the jobs manager when a job is created.
"""
def on_job_updated(self, job):
""" Method called when a change occurs in a job attribute.
"""
def on_job_purged(self, job):
""" Method called by the jobs manager when a job is purged.
"""
cc-common-v8/cloudcontrol/common/jobs/job.py 0000664 0000000 0000000 00000021552 12600735274 0021357 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Job and job state.
This module contains the following objects:
- JobState: A class representing the job state
- Job: A class (subclass of Thread) representing the job itself
- JobCancelError, FreezedJobError, AlreadyGotAttachmentError: exceptions used by
job classes.
"""
import logging
from datetime import datetime
from threading import Thread
class JobCancelError(Exception):
""" Error raised to cancel a job.
"""
pass
class FreezedJobError(Exception):
""" Error raised when the requested action is not compatible with the
freeze state of the job.
"""
pass
class AlreadyGotAttachmentError(Exception):
""" Error raised when an attachment is requested more than one time.
"""
pass
class JobState(object):
""" State of a job.
A system job is a special job which can be used to start background tasks.
System jobs are not persistent and are not notified through the interface.
This class represent the full state of a job.
"""
STATES = ('init', 'running', 'done', 'cancelling', 'rollbacking')
def __init__(self, logger, job_class, job_id, manager, owner, settings,
system=False, batch=None):
self.logger = logger
self._frozen = False # Is the jobstate frozen (loaded from store and
# no more modifiable)
self._manager = manager
# Jobs attributes:
self._job_id = job_id
self._title = str(job_class.__class__) # The title of the job
self._status = None # Textual description about status
self._state = 'init' # State of the job
self._owner = owner # Id of the owner object
self._created = datetime.now() # Creation date of the job
self._ended = None # Termination date of the job
self._system = system # Is a system job?
self._attachments = set() # Files attached to the job
self._batch = batch # Batch identifier
# Bound to the job itself:
self._job = job_class(self.logger, self, **settings)
def _check_freeze(self):
""" Raise FreezedJobError if object is freezed.
"""
if self._frozen:
raise FreezedJobError('This job is freezed')
def __getstate__(self):
jobstate = self.__dict__.copy()
jobstate['logger'] = None
jobstate['_job'] = None
jobstate['_manager'] = None
jobstate['_frozen'] = True
if jobstate['_state'] != 'done':
jobstate['_status'] = 'Interrupted while: %s' % jobstate['_status']
jobstate['_ended'] = datetime.now()
jobstate['_state'] = 'done'
return jobstate
def _notify_attrs_changes(self):
if self._manager is not None:
self._manager.notify_attrs_changes(self)
@property
def manager(self):
return self._manager
@manager.setter
def manager(self, value):
if not self._frozen:
raise FreezedJobError('manager can be set only when job is frozen')
else:
self._manager = value
#
# Attribute properties.
#
@property
def id(self):
return self._job_id
@property
def system(self):
return self._system
@property
def batch(self):
return self._batch
@property
def title(self):
return self._title
@title.setter
def title(self, value):
self._check_freeze()
self._title = value
self._notify_attrs_changes()
@property
def status(self):
return self._status
@status.setter
def status(self, value):
self._check_freeze()
self.logger.info('Status: %s' % value)
self._status = value
self._notify_attrs_changes()
@property
def state(self):
return self._state
@state.setter
def state(self, value):
self._check_freeze()
if value not in self.STATES:
raise ValueError('Bad state')
else:
self.logger.info('State: %s' % value)
self._state = value
if value == 'done':
self._job = None
self._notify_attrs_changes()
@property
def created(self):
return self._created
@property
def ended(self):
return self._ended
@property
def owner(self):
return self._owner
@property
def attachments(self):
return self._attachments
def start(self):
""" Start the job.
"""
# Configure logger of the job:
if not self.system:
handler = logging.StreamHandler(self.attachment('logs'))
fmt = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
handler.setFormatter(fmt)
self.logger.addHandler(handler)
self._job.start()
def report(self, status, state=None):
""" Report the job status.
"""
self._check_freeze()
self.status = status
if state is not None:
self.state = state
if state == 'done':
self._ended = datetime.now()
self._notify_attrs_changes()
def attachment(self, name):
if name in self._attachments:
raise AlreadyGotAttachmentError('This attachment already exists')
fattach = self._manager.get_attachment_file(self, name)
self._attachments.add(name)
self._notify_attrs_changes()
return fattach
def cancel(self):
""" Cancel the job.
"""
self._check_freeze()
if self.state != 'done':
self.state = 'cancelling'
def read_attachment(self, name):
if name not in self._attachments:
raise KeyError('Unknown attachment')
return self.manager.read_attachment(self, name)
class Job(Thread, object):
""" Base class for a job.
"""
daemon = True # Set job as a daemon thread
def __init__(self, logger, state, **settings):
Thread.__init__(self)
self.logger = logger
self.state = state
self._settings = settings
# Actions to do by the rollback method:
self._wayback = []
# Set the thread name:
self.name = self.state.id
def _rollback(self, error):
""" Rollback the job using the specified error message.
"""
self.report('Rollback in progress', state='rollbacking')
try:
for func in reversed(self._wayback):
func()
except Exception as err:
self.report('Rollback failed: %s (error was: %s)' % err, error, state='done')
else:
self.report('Cancelled: %s' % error, state='done')
#
# Shortcuts to useful state methods:
#
def report(self, *args, **kwargs):
return self.state.report(*args, **kwargs)
def attachment(self, *args, **kwargs):
return self.state.attachment(*args, **kwargs)
@property
def title(self):
return self.state.title
@title.setter
def title(self, value):
self.state.title = value
#
# Public methods:
#
def run(self, *args, **kwargs):
""" Run the job.
"""
self.state.state = 'running'
try:
self.job(**self._settings)
except JobCancelError as err:
self._rollback(str(err))
self.logger.info('Job cancelled')
except Exception as err:
self.logger.exception('Error while executing job: %r(%s)', err, err)
self._rollback(str(err))
else:
self.report('Success', state='done')
def checkpoint(self, wayback_callback=None):
""" Check if job is not cancelled, else raise the CancelJobError. Also
optionally add the provided callback to the wayback list.
"""
if self.state.state == 'cancelling':
raise JobCancelError('Job has been cancelled by user')
if wayback_callback is not None:
self._wayback.append(wayback_callback)
def job(self, *args, **kwargs):
""" Method to override in order to define the job behavior.
"""
pass
cc-common-v8/cloudcontrol/common/jobs/manager.py 0000664 0000000 0000000 00000006110 12600735274 0022210 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Jobs manager class.
"""
from cloudcontrol.common.jobs.helpers import AutoIncrementCounter
from cloudcontrol.common.jobs.job import JobState
class JobsManager(object):
""" Store created jobs.
"""
def __init__(self, logger, interface, store):
self.logger = logger
self._jobs = {}
self._interface = interface
self._store = store
self._counter = AutoIncrementCounter(store)
# Load jobs from store:
for job in self._store.iter_jobs():
job.manager = self
self._jobs[job.id] = job
self._interface.on_job_created(job)
self._interface.on_job_updated(job)
#
# Jobs methods
#
def notify_attrs_changes(self, job):
""" Notify a change in attributes of a job to the manager.
"""
if not job.system:
self._interface.on_job_updated(job)
self._store.update_job(job)
def get_attachment_file(self, job, name):
""" Get an attachment file object open with write mode for the specified
job.
"""
if not job.system:
return self._store.get_attachment_file(job.id, name)
def read_attachment(self, job, name):
""" Get content of an attachment in a string.
"""
if not job.system:
return self._store.read_attachment(job.id, name)
#
# Public methods
#
def spawn(self, job_class, owner, system=False, batch=None, settings={}):
""" Spawn a new job.
"""
job_id = 'job-%s' % self._counter.get()
job_logger = self.logger.getChild(str(job_id))
job = JobState(job_logger, job_class, job_id, self, owner,
settings=settings, system=system, batch=batch)
if not system:
self._store.create_job(job)
self._interface.on_job_created(job)
self._jobs[job_id] = job
job.start()
self.logger.info('Started new job #%s of class %s', job_id, job_class)
return job
def get(self, job_id):
""" Get a job by its id.
"""
return self._jobs[job_id]
def purge(self, job_id):
""" Purge a done job.
"""
job = self.get(job_id)
if job.state != 'done':
raise Exception('Job is not done')
self._store.delete_job(job_id)
del self._jobs[job.id]
self._interface.on_job_purged(job.id)
cc-common-v8/cloudcontrol/common/jobs/store.py 0000664 0000000 0000000 00000006331 12600735274 0021737 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Manage the persistence of jobs on disk.
"""
import os
import shutil
import glob
import cPickle as pickle
class JobsStore(object):
def __init__(self, base_directory):
self._base_directory = base_directory
# Create the base directory if it doesn't exists yet:
try:
os.mkdir(base_directory)
except OSError as err:
if err.errno != 17:
raise
def _get_job_dir(self, job_id):
return os.path.join(self._base_directory, job_id)
def _get_attrs_dir(self, job_id):
return os.path.join(self._get_job_dir(job_id), 'job.dat')
def _get_counter_file(self):
return os.path.join(self._base_directory, 'counter')
def _get_attachment_file(self, job_id, name):
return os.path.join(self._get_job_dir(job_id), '%s.attachment' % name)
def store_id_counter(self, current):
with open(self._get_counter_file(), 'w') as fcounter:
pickle.dump(current, fcounter)
def get_id_counter(self):
try:
with open(self._get_counter_file(), 'r') as fcounter:
return pickle.load(fcounter)
except IOError as err:
if err.errno != 2:
raise
return 0
def create_job(self, job):
""" Create a new job.
"""
try:
os.mkdir(self._get_job_dir(job.id))
except OSError as err:
if err.errno != 17:
raise
else:
self.update_job(job)
def update_job(self, job):
""" Update attributes of a job.
"""
with open(self._get_attrs_dir(job.id), 'w') as fjob:
pickle.dump(job, fjob)
def delete_job(self, job_id):
""" Delete a job.
"""
shutil.rmtree(self._get_job_dir(job_id))
def iter_jobs(self):
""" Iterate over stored jobs.
"""
for filename in glob.glob(os.path.join(self._base_directory, 'job-*', 'job.dat')):
with open(filename, 'r') as fjob:
yield pickle.load(fjob)
def get_attachment_file(self, job_id, name, append=False):
""" Get an open-writable file object for an attachment.
"""
mode = 'a' if append else 'w'
return open(self._get_attachment_file(job_id, name), mode)
def read_attachment(self, job_id, name):
""" Get the content of an attachment in a string.
"""
with open(self._get_attachment_file(job_id, name), 'r') as fattach:
return fattach.read() #FIXME: DANGEROUS IF THE FILE IS BIG LIKE UR MOM
cc-common-v8/cloudcontrol/common/tql/ 0000775 0000000 0000000 00000000000 12600735274 0020071 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/tql/__init__.py 0000664 0000000 0000000 00000001266 12600735274 0022207 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
cc-common-v8/cloudcontrol/common/tql/db/ 0000775 0000000 0000000 00000000000 12600735274 0020456 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/tql/db/__init__.py 0000664 0000000 0000000 00000001266 12600735274 0022574 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
cc-common-v8/cloudcontrol/common/tql/db/db.py 0000664 0000000 0000000 00000033653 12600735274 0021427 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Main class of the TQL database.
"""
import re
import time
from copy import copy
from fnmatch import fnmatch
from functools import partial
import ply.yacc as yacc
from cloudcontrol.common.tql.parser.parser import TqlParser
from cloudcontrol.common.tql.db.object import TqlObject
from cloudcontrol.common.tql.db.requestor import StaticRequestor
from cloudcontrol.common.datastructures.ordereddict import OrderedDict
from cloudcontrol.common.tql.parser.ast import (Filter, FilterPresence,
UnionOperator,
IntersectionOperator,
DifferenceOperator,
ShowOperator, LimitOperator,
SortOperator)
RE_NATURAL_SORTING = re.compile('([0-9]+|[^0-9]+)')
def natural_tag_sorter(tag, obj):
""" Natural sorter for a tag in an object.
"""
value = obj.get(tag) if tag in obj else ''
return tuple(int(x) if x.isdigit() else x
for x in RE_NATURAL_SORTING.findall(value))
#
# Response objects:
#
class TqlResponseObject(object):
""" A proxy for an object returned after a query.
"""
def __init__(self, object_to_wrap):
self.object = object_to_wrap
self._show = set(('id',))
self._tags = {} # Mapping between fetched tags and its value
def __repr__(self):
return '' % self.object
#
# Bind special methods:
#
def __contains__(self, value):
return self.object.__contains__(value)
def __hash__(self):
return self.object.__hash__()
def __cmp__(self, other):
return cmp(self.object, other.object)
def __getitem__(self, name):
return self.object.__getitem__(name)
def __iter__(self):
return self.object.__iter__()
def __getattr__(self, name):
attr = getattr(super(TqlResponseObject, self).__getattribute__('object'), name)
if attr is None:
raise AttributeError('%r object has no attribute %r' % (self.__class__, name))
else:
return attr
def set(self, name, value):
""" Set a response value for the specified tag.
"""
self._tags[name] = value
def get(self, name):
""" Get a response value for the specified tag.
"""
return self._tags[name]
def copy(self):
return copy(self)
@property
def show(self):
return self._show
@show.setter
def show(self, show):
self._show = show
@property
def show_tags(self):
for tag in self.show:
tag = self.object.get(tag)
if tag is not None:
yield tag
def to_dict(self, tags=None):
if tags is None:
tags = self._show
return dict((k, unicode(v)) for k, v in self._tags.iteritems() if k in tags)
class TqlResponse(object):
""" A set of TqlResponseObject returned after a query.
"""
def __init__(self, requestor, copy_from=None):
self._requestor = requestor
if copy_from is None:
self._objects = OrderedDict()
else:
self._objects = OrderedDict((obj.id, obj) for obj in copy_from)
def __repr__(self):
return '' % ', '.join(self.objects)
def __iter__(self):
return self._objects.itervalues()
def __contains__(self, obj):
return self.has(obj)
def __and__(self, other):
response = TqlResponse(self._requestor)
for obj in self:
if obj in other:
obj = obj.copy()
obj.show |= self[obj.id].show
response.add(obj)
return response
def __or__(self, other):
response = TqlResponse(self._requestor)
for obj in other:
if obj in self:
obj = obj.copy()
obj.show |= self[obj.id].show
response.add(obj)
for obj in self:
if obj not in response:
response.add(obj)
return response
def __sub__(self, other):
response = self.copy()
for obj in other:
try:
response.remove(obj)
except KeyError:
pass
return response
def __len__(self):
return len(self.objects)
def __getitem__(self, name):
obj = self.get(name)
if obj is None:
raise KeyError('Unknown object %s' % name)
return obj
@property
def objects(self):
""" Get the dict of object stored in this response.
"""
return self._objects
def set_requestor(self, requestor):
""" Set a new requestor on this TqlResponse object.
"""
self._requestor = requestor
def get(self, object_id, default=None):
""" Return the TqlResponseObject with the specified id, or default.
"""
if isinstance(object_id, TqlResponseObject):
object_id = object_id.id
return self._objects.get(object_id, default)
def has(self, object_id):
""" Return True if the provided id (or TqlResponseObject) is found in
this TqlResponse object.
"""
return self.get(object_id) is not None
def copy(self):
""" Copy this TqlResponse object into another one.
"""
return TqlResponse(self._requestor, self)
def limit(self, start, end, step=None):
""" Return a new TqlResponse with objects inside limit.
"""
return TqlResponse(self._requestor, list(self)[start:end:step])
def sorted(self, tag, reverse=False):
""" Return a new TqlResponse with objects sorted by provided tag.
"""
self.fetch(tag)
sorter = partial(natural_tag_sorter, tag)
objects = tuple(sorted(self._objects.itervalues(), key=sorter, reverse=reverse))
for obj in objects:
obj.show.add(tag)
return TqlResponse(self._requestor, objects)
def filter(self, tag, cmp_func):
""" Filter value of specified tag using the provided comparison function.
"""
self.fetch(tag)
matching = TqlResponse(self._requestor)
for obj in self._objects.itervalues():
try:
tag_value = obj.get(tag)
except KeyError:
continue
else:
if cmp_func(tag_value):
matching.add(obj)
obj.show.add(tag)
return matching
def fetch(self, tags):
""" Fetch some tags.
"""
if isinstance(tags, basestring):
tags = (tags,)
self._requestor.fetch(self._objects.itervalues(), tags)
def add(self, obj):
""" Add an object in the response.
:param obj: the object to add to the response
"""
if obj not in self._objects:
self._objects[obj.id] = obj
def remove(self, obj):
""" Remove an object in the response.
:param obj: the object to remove from the response
"""
del self._objects[obj.id]
#
# Database:
#
class TqlDatabaseError(Exception):
""" Exception raised when an error occurs while a tql database operation.
"""
class TqlDatabase(object):
def __init__(self, default_requestor=None):
self._objects = {}
self._stats_maxquerytime = 0
self._stats_numquery = 0
if default_requestor is None:
self._default_requestor = StaticRequestor()
else:
self._default_requestor = default_requestor
@property
def objects(self):
return self._objects.itervalues()
def register(self, obj):
""" Register a new object in the database.
"""
if obj.id not in self._objects:
self._objects[obj.id] = obj
else:
raise TqlDatabaseError('An object is already registered with this id')
def unregister(self, obj):
""" Unregister an object from the database.
"""
if isinstance(obj, TqlObject):
obj = obj.id
del self._objects[obj]
def stats(self):
""" Get statistics about the database.
:return: return a dict of statistics
"""
stats = {}
stats['objects'] = len(self._objects)
stats['max_query_time'] = self._stats_maxquerytime
stats['queries'] = self._stats_numquery
return stats
def get(self, object_id):
return self._objects[object_id]
def raw_query(self, tql, requestor=None):
started = time.time()
self._stats_numquery += 1
if requestor is None:
requestor = self._default_requestor
if isinstance(tql, basestring):
tql = TqlParser(tql, debug=False, write_tables=False,
errorlog=yacc.NullLogger()).parse()
evaluated = self._evaluate_ast(tql, requestor)
duration = time.time() - started
if duration > self._stats_maxquerytime:
self._stats_maxquerytime = duration
return evaluated
def query(self, tql, show=None, requestor=None):
if requestor is None:
requestor = self._default_requestor
result = self.raw_query(tql, requestor)
objects = []
if show is None:
show = ()
for obj in result:
tags_to_show = set(obj.itermatchingtags(show)) | set(obj.show_tags)
requestor.fetch((obj,), [t.name for t in tags_to_show])
objects.append(obj.to_dict([t.name for t in tags_to_show]))
return objects
def get_by_id(self, object_id, tags=None, requestor=None):
""" Get an object result including specified tags (or all if
not specified).
"""
if requestor is None:
requestor = self._default_requestor
if tags is None:
tags = ('*',)
obj = self._objects.get(object_id)
if obj is None:
return None
tags = [t.name for t in obj.itermatchingtags(tags)]
robj = TqlResponseObject(obj)
requestor.fetch((robj,), tags)
return robj.to_dict(tags)
#
# TQL AST evaluation:
#
def _evaluate_ast(self, ast, requestor):
objects = TqlResponse(requestor, (TqlResponseObject(obj) for obj in self._objects.values()))
return self._evaluate_expression(objects, ast.expression)
def _evaluate_expression(self, objects, expression):
if isinstance(expression, Filter):
return self._evaluate_filter(objects, expression)
elif isinstance(expression, FilterPresence):
return self._evaluate_filter_presence(objects, expression)
elif isinstance(expression, UnionOperator):
return self._evaluate_union(objects, expression)
elif isinstance(expression, IntersectionOperator):
return self._evaluate_intersection(objects, expression)
elif isinstance(expression, DifferenceOperator):
return self._evaluate_difference(objects, expression)
elif isinstance(expression, ShowOperator):
return self._evaluate_show(objects, expression)
elif isinstance(expression, SortOperator):
return self._evaluate_sort(objects, expression)
elif isinstance(expression, LimitOperator):
return self._evaluate_limit(objects, expression)
def _evaluate_filter(self, objects, filter):
return objects.filter(filter.name, filter.match)
def _evaluate_filter_presence(self, objects, filter):
matching = TqlResponse(objects._requestor)
for obj in objects:
if (filter.name in obj and not filter.invert) or (filter.name not in obj and filter.invert):
matching.add(obj)
obj.show.add(filter.name)
return matching
def _evaluate_union(self, objects, union):
left = self._evaluate_expression(objects, union.left_expression)
right = self._evaluate_expression(objects, union.right_expression)
return left | right
def _evaluate_intersection(self, objects, intersection):
left = self._evaluate_expression(objects, intersection.left_expression)
right = self._evaluate_expression(objects, intersection.right_expression)
return left & right
def _evaluate_difference(self, objects, difference):
left = self._evaluate_expression(objects, difference.left_expression)
right = self._evaluate_expression(objects, difference.right_expression)
return left - right
def _evaluate_show(self, objects, show):
objects = self._evaluate_expression(objects, show.expression)
for obj in objects:
if show.invert:
for tag in obj.object:
if fnmatch(tag.name, show.pattern):
obj.show.discard(tag.name)
else:
for tag in obj.object:
if fnmatch(tag.name, show.pattern):
obj.show.add(tag.name)
return objects
def _evaluate_sort(self, objects, sorter):
objects = self._evaluate_expression(objects, sorter.expression)
return objects.sorted(sorter.name, sorter.invert)
def _evaluate_limit(self, objects, limit):
objects = self._evaluate_expression(objects, limit.expression)
return objects.limit(limit.start, limit.stop)
cc-common-v8/cloudcontrol/common/tql/db/helpers.py 0000664 0000000 0000000 00000002641 12600735274 0022475 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Some helpers for the TQL database.
"""
import time
from datetime import datetime, timedelta
def taggify(value):
""" Transform any value into a tag value.
"""
if isinstance(value, bool):
value = 'yes' if value else 'no'
elif isinstance(value, datetime):
value = str(int(time.mktime(value.timetuple())))
elif isinstance(value, timedelta):
value = str(value.seconds + value.days * 86400)
elif isinstance(value, (set, list, tuple)):
value = ' '.join((str(x) for x in value))
elif isinstance(value, dict):
value = ' '.join((':'.join((str(k), str(v))) for k, v in value.iteritems()))
elif value is None:
pass
else:
value = str(value)
return value
cc-common-v8/cloudcontrol/common/tql/db/object.py 0000664 0000000 0000000 00000005434 12600735274 0022304 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Objects as stored in the Tql database.
"""
from fnmatch import fnmatch
from cloudcontrol.common.tql.db.tag import StaticTag
class TqlObject(object):
""" An object in the TQL database.
"""
def __init__(self, id_):
id_ = StaticTag('id', id_)
self._tags = {'id': id_}
def __repr__(self):
return '' % self.id
def __contains__(self, value):
return value in self._tags
def __hash__(self):
return hash(self._tags['id'].value)
def __cmp__(self, other):
return cmp(self.id, other.id)
def __getitem__(self, name):
return self._tags[name]
def __iter__(self):
return iter(self.itertags())
@property
def id(self):
""" Get the id of the object.
"""
return self._tags['id'].value
def itertags(self):
""" Iter over tags registered on this object.
"""
for tag in self._tags.values():
yield tag
def itermatchingtags(self, patterns):
""" Iter over tags matching given patterns.
"""
if isinstance(patterns, basestring):
patterns = (patterns,)
for tag in self._tags.values():
for pattern in patterns:
if fnmatch(tag.name, pattern):
yield tag
break
def get(self, name, default=None):
""" Get the tag with the specified name (or default).
"""
return self._tags.get(name, default)
def get_value(self, name, default=None):
""" Get the value of the tag with the specified name (or default).
"""
tag = self.get(name, None)
return tag if tag is not None else default
def register(self, tag):
""" Register a tag on this object.
"""
if tag.name in self._tags:
raise KeyError('A tag with this name is '
'already registered on this object')
else:
self._tags[tag.name] = tag
def unregister(self, name):
""" Unregister the tag with specified name.
"""
del self._tags[name]
cc-common-v8/cloudcontrol/common/tql/db/requestor.py 0000664 0000000 0000000 00000006120 12600735274 0023060 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Requestors are components in charge of updating list of tags on a list of
objects.
"""
from collections import defaultdict
from cloudcontrol.common.tql.db.tag import StaticTagInterface
def fetcher(*args):
""" Decorator used to create a fetching method.
"""
def decorator(func):
func._fetcher_interfaces = args
return func
return decorator
class BaseRequestor(object):
""" Base class for all requestors.
"""
def __init__(self):
self._fetcher_interface_mapping = {}
for attr_name in dir(self):
attr = getattr(self, attr_name)
interfaces = getattr(attr, '_fetcher_interfaces', None)
if interfaces is not None:
for interface in interfaces:
if interface in self._fetcher_interface_mapping:
continue
self._fetcher_interface_mapping[interface] = attr
def fetch(self, objects, tags):
""" This method allow to fetch tags on objects.
"""
map = defaultdict(lambda: [])
for obj in objects:
for tag_name in tags:
try:
tag = obj[tag_name]
except KeyError:
continue
map[obj].append(tag)
return self.fetch_map(map)
def fetch_map(self, map):
""" Fetch tags for specified objects.
"""
# Dispatch each tag to a fetcher according to its type:
fetcher_map = defaultdict(lambda: defaultdict(lambda: []))
for obj, tags in map.iteritems():
for tag in tags:
# Search the matching fetcher:
for interface, fetcher in self._fetcher_interface_mapping.iteritems():
if isinstance(tag, interface):
fetcher_map[fetcher][obj].append(tag)
else:
pass #XXX: fetcher not found
# Call each fetcher:
for fetcher, map in fetcher_map.iteritems():
fetcher(map)
class StaticRequestor(BaseRequestor):
""" A requestion which is able to request value for tags implementing the
static interface.
"""
@fetcher(StaticTagInterface)
def fetch_static(self, map):
""" Fetch tag using the StaticTagInterface.
"""
for obj, tags in map.iteritems():
for tag in tags:
obj.set(tag.name, tag.value)
cc-common-v8/cloudcontrol/common/tql/db/tag.py 0000664 0000000 0000000 00000010727 12600735274 0021612 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" In-database tags representation.
"""
from abc import ABCMeta, abstractproperty
from datetime import datetime, timedelta
class BaseTagInterface(object):
""" Base class for all tag interfaces.
"""
__metaclass__ = ABCMeta
class BaseTag(object):
""" Base class for a tag in the database.
:param name: name of the tag
"""
interface = None # Define the tag fetching interface of the tag
def __init__(self, name, **kwargs):
self._name = name
def __repr__(self):
return '<%s %s>' % (self.__class__.__name__, self.name)
def __hash__(self):
return hash(self.name)
#
# Publics:
#
@property
def name(self):
return self._name
#
# Static tags
#
class StaticTagInterface(BaseTagInterface):
""" An interface used to get static value stored by tags.
"""
@abstractproperty
def value(self):
""" The value of the tag.
"""
@value.setter
def value(self, value):
""" Set the value of the tag.
"""
class StaticTag(BaseTag):
""" Static tag in the database.
"""
interface = 'static'
def __init__(self, name, value='', **kwargs):
super(StaticTag, self).__init__(name, **kwargs)
self._value = value
def __repr__(self):
return '<%s %s (value=%r)>' % (self.__class__.__name__, self.name, self.value)
#
# Implentation of StaticTagInterface:
#
@property
def value(self):
return self._value
@value.setter
def value(self, value):
self._value = value
StaticTagInterface.register(StaticTag)
class CallbackTag(BaseTag, StaticTagInterface):
""" A tag which is defined after a call to a callback.
"""
interface = 'static'
def __init__(self, name, callback, ttl=None, extra={}, **kwargs):
super(CallbackTag, self).__init__(name, **kwargs)
self._callback = callback
self._callback_extra = extra
self._ttl = ttl
self._cache_value = ''
self._cache_last_update = None
def __repr__(self):
if self._cache_last_update is not None and self._ttl is not None:
dt = self.ttl - (datetime.now() - self._cache_last_update)
expire = (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 10**6) / 10.0**6
else:
expire = None
return '<%s %s (ttl=%r expire=%r cached=%r cb=%r)>' % (self.__class__.__name__,
self.name,
self._ttl,
expire,
self._cache_value,
self._callback.__name__)
#
# Implentation of StaticTagInterface:
#
@property
def value(self):
# Update the cached value if it has expired or not retrieved yet:
if self.ttl is not None or (self.ttl is None and self._cache_last_update is None):
if self._cache_last_update is None or datetime.now() - self._cache_last_update > self.ttl:
self.value = str(self.callback(**self._callback_extra))
return self._cache_value
@value.setter
def value(self, value):
self._cache_last_update = datetime.now()
self._cache_value = value
#
# Publics:
#
@property
def callback(self):
return self._callback
@callback.setter
def callback(self, callback):
self._callback = callback
@property
def ttl(self):
return timedelta(seconds=self._ttl) if self._ttl is not None else None
@ttl.setter
def ttl(self, ttl):
self._ttl = ttl
def invalidate(self):
self._cache_last_update = None
cc-common-v8/cloudcontrol/common/tql/parser/ 0000775 0000000 0000000 00000000000 12600735274 0021365 5 ustar 00root root 0000000 0000000 cc-common-v8/cloudcontrol/common/tql/parser/__init__.py 0000664 0000000 0000000 00000001266 12600735274 0023503 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
cc-common-v8/cloudcontrol/common/tql/parser/ast.py 0000664 0000000 0000000 00000010273 12600735274 0022531 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" TQL ast classes.
"""
import re
from fnmatch import fnmatch
class TqlAst(object):
""" The AST root.
"""
def __init__(self, expression=None):
self.expression = expression
class Filter(object):
""" A filter.
"""
def __init__(self, name, filter=None, value=None, neg=False):
self.name = name
self.filter = filter
self.value = value
try:
self.value = int(value)
except ValueError:
try:
self.value = float(value)
except ValueError:
self.is_number = False
else:
self.is_number = True
else:
self.is_number = True
self.neg = neg
def match(self, value):
if self.is_number:
# If provided filter value is a number, we must try to cast the tag
# value as a number to do a value comparison:
try:
value = int(value)
except ValueError:
try:
value = float(value)
except ValueError:
pass
match = False
if self.filter == '=' and value == self.value:
match = True
elif self.filter == '>' and value > self.value:
match = True
elif self.filter == '<' and value < self.value:
match = True
elif self.filter == '>=' and value >= self.value:
match = True
elif self.filter == '<=' and value <= self.value:
match = True
elif self.filter == ':' and fnmatch(str(value), str(self.value)):
match = True
elif self.filter == '~' and re.match(str(self.value), str(value)):
match = True
if self.neg:
match = not match
return match
class FilterPresence(object):
""" A filter for tag presence in object.
"""
def __init__(self, name):
if name.startswith('-'):
name = name[1:]
self.invert = True
else:
self.invert = False
self.name = name
class BinaryOperator(object):
""" Base class for binary operators.
"""
def __init__(self, left_expression, right_expression):
self.left_expression = left_expression
self.right_expression = right_expression
class UnionOperator(BinaryOperator):
""" An union operator (eg: expr|expr).
"""
SYMBOL = '|'
class IntersectionOperator(BinaryOperator):
""" An intersection operator (eg: expr&expr).
"""
SYMBOL = '&'
class DifferenceOperator(BinaryOperator):
""" A difference operator (eg: expr/expr).
"""
SYMBOL = '/'
class ShowOperator(object):
""" A show operator (eg: $tag).
"""
def __init__(self, expression, pattern):
self.expression = expression
if pattern.startswith('-'):
pattern = pattern[1:]
self.invert = True
else:
self.invert = False
self.pattern = pattern
class SortOperator(object):
""" A sorting operator (eg: %tag).
"""
def __init__(self, expression, name):
self.expression = expression
if name.startswith('-'):
name = name[1:]
self.invert = True
else:
self.invert = False
self.name = name
class LimitOperator(object):
""" A limitation operator (eg: ^1:100).
"""
def __init__(self, expression, start=None, stop=None):
self.expression = expression
self.start = start
self.stop = stop
cc-common-v8/cloudcontrol/common/tql/parser/errors.py 0000664 0000000 0000000 00000001732 12600735274 0023256 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" Exception raised while the parsing process.
"""
class ParsingError(Exception):
""" Error raised on a parsing error.
"""
def __init__(self, msg, line=None, column=None):
super(ParsingError, self).__init__(msg)
self.line = line
self.column = column
cc-common-v8/cloudcontrol/common/tql/parser/parser.py 0000664 0000000 0000000 00000017016 12600735274 0023240 0 ustar 00root root 0000000 0000000 # This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
""" The TQL lexer and parser.
"""
import re
import ply.lex as lex
import ply.yacc as yacc
from ast import (TqlAst, Filter, FilterPresence, UnionOperator,
IntersectionOperator, DifferenceOperator, ShowOperator,
SortOperator, LimitOperator)
from errors import ParsingError
UNITS = {'k': 10 ** 3,
'm': 10 ** 6,
'g': 10 ** 9,
't': 10 ** 12,
'p': 10 ** 15,
'e': 10 ** 18,
'z': 10 ** 21,
'y': 10 ** 24,
'ki': 2 ** 10,
'mi': 2 ** 20,
'gi': 2 ** 30,
'ti': 2 ** 40,
'pi': 2 ** 50,
'ei': 2 ** 60,
'zi': 2 ** 70,
'yi': 2 ** 80}
RE_UNIT = re.compile(r'(?P[-+]?[0-9]+(\.[0-9]+)?)[ ]?'
'(?P%s)$' % '|'.join(UNITS), re.IGNORECASE)
class TqlLexer(object):
""" Lexer for the TQL format.
"""
def __init__(self, **kwargs):
self._lexer = lex.lex(module=self, **kwargs)
#
# Tokens
#
tokens = ('UNION', 'INTERSECTION', 'DIFFERENCE', 'CIRCUMFLEX', 'DOLLAR',
'PERCENT', 'EQUAL', 'COLON', 'TILDE', 'GT', 'GTE', 'LT', 'LTE',
'NOT', 'WORD', 'TEXT', 'LEFT_PAR', 'RIGHT_PAR', 'EOL')
t_UNION = '\|'
t_INTERSECTION = '&'
t_DIFFERENCE = '/'
t_CIRCUMFLEX = '\^'
t_DOLLAR = '\$'
t_PERCENT = '%'
t_EQUAL = '='
t_COLON = ':'
t_TILDE = '~'
t_GT = '>'
t_GTE = '>='
t_LT = '<'
t_LTE = '<='
t_NOT = '!'
t_WORD = '[^ |&^$%=:~>>=<<=!()/]+'
t_LEFT_PAR = '\('
t_RIGHT_PAR = '\)'
t_ignore = ' \t'
def t_TEXT(self, token):
r'(["]([\\]["]|[^"]|)*["]|[\']([\\][\']|[^\'])*[\'])'
value = token.value[1:-1].replace('\\' + token.value[0], token.value[0])
token.value = value
return token
def t_EOL(self, token):
r'[\n]+'
token.lexer.lineno += len(token.value)
def t_error(self, token):
raise ParsingError('Illegal character %r' % token.value[0],
line=self._lexer.lineno, column=self._lexer.lexpos)
#
# Public methods
#
def column(self, lexpos):
""" Find the column according to the lexpos.
"""
# This code is taken from the python-ply documentation
# see: http://www.dabeaz.com/ply/ply.html section 4.6
last_cr = self._current_input.rfind('\n', 0, lexpos)
if last_cr < 0:
last_cr = 0
column = (lexpos - last_cr)
return column
#
# Bindings to the internal _lexer object
#
def input(self, input):
self._current_input = input
return self._lexer.input(input)
def __getattr__(self, name):
lexer = super(TqlLexer, self).__getattribute__('_lexer')
attr = getattr(lexer, name)
if attr is None:
raise AttributeError("'%s' object has no attribute '%s'" % (self, name))
else:
return attr
class TqlParser(object):
""" Parser for the TQL format.
"""
tokens = TqlLexer.tokens
precedence = (('left', 'UNION', 'INTERSECTION'),
('right', 'CIRCUMFLEX', 'DOLLAR', 'PERCENT'))
def __init__(self, input, **kwargs):
self._input = input
self._input_name = kwargs.pop('input_name', '')
self._lexer = kwargs.pop('lexer', TqlLexer())
self._parser = yacc.yacc(module=self, **kwargs)
#
# Rules
#
start = 'input'
def p_input(self, p):
"""input : expression
|"""
if len(p) == 1:
p[0] = TqlAst()
else:
p[0] = TqlAst(p[1])
def p_value(self, p):
"""value : expanded_word
| TEXT"""
p[0] = p[1]
def p_expanded_word(self, p):
""" expanded_word : WORD"""
match = RE_UNIT.match(p[1])
if match:
try:
num = int(match.group('num'))
except ValueError:
num = float(match.group('num'))
word = num * UNITS[match.group('unit').lower()]
else:
word = p[1]
p[0] = word
def p_expression_filter(self, p):
"""expression : WORD filter value
| WORD NOT filter value"""
if len(p) == 4:
p[0] = Filter(p[1], p[2], p[3])
elif len(p) == 5:
p[0] = Filter(p[1], p[3], p[4], neg=True)
def p_expression_filter_presence(self, p):
"""expression : WORD"""
p[0] = FilterPresence(p[1])
def p_expression_par(self, p):
"""expression : LEFT_PAR expression RIGHT_PAR"""
p[0] = p[2]
def p_expression_binary(self, p):
"""expression : expression UNION expression
| expression INTERSECTION expression
| expression DIFFERENCE expression"""
if p[2] == '|':
p[0] = UnionOperator(p[1], p[3])
elif p[2] == '&':
p[0] = IntersectionOperator(p[1], p[3])
else:
p[0] = DifferenceOperator(p[1], p[3])
def p_expression_show(self, p):
"""expression : expression DOLLAR WORD"""
p[0] = ShowOperator(p[1], p[3])
def p_expression_sort(self, p):
"""expression : expression PERCENT WORD"""
p[0] = SortOperator(p[1], p[3])
def p_expression_limit(self, p):
"""expression : expression CIRCUMFLEX WORD
| expression CIRCUMFLEX COLON WORD
| expression CIRCUMFLEX WORD COLON WORD"""
try:
if len(p) == 4:
p[0] = LimitOperator(p[1], stop=int(p[3]))
elif len(p) == 5:
p[0] = LimitOperator(p[1], start=-int(p[4]))
elif len(p) == 6:
p[0] = LimitOperator(p[1], start=int(p[3]), stop=int(p[5]))
except ValueError:
column = self._lexer.column(p.lexpos(2))
raise ParsingError('Syntax error near of "^": limit argument '
'must be an integer', column=column)
def p_filter(self, p):
"""filter : EQUAL
| COLON
| TILDE
| GT
| GTE
| LT
| LTE"""
p[0] = p[1]
def p_error(self, token):
if token is None:
raise ParsingError('Syntax error near of "EOL"')
else:
column = self._lexer.column(token.lexpos)
raise ParsingError('Syntax error near of "%s"' % token.value, column=column)
#
# Bindings to the internel _parser object
#
def parse(self):
return self._parser.parse(self._input, self._lexer, tracking=True)
def __getattr__(self, name):
parser = super(TqlParser, self).__getattribute__('_parser')
attr = getattr(parser, name)
if attr is None:
raise AttributeError("'%s' object has no attribute '%s'" % (self, name))
else:
return attr
cc-common-v8/setup.py 0000664 0000000 0000000 00000002431 12600735274 0015004 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python
# This file is part of CloudControl.
#
# CloudControl is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# CloudControl is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with CloudControl. If not, see .
from setuptools import setup, find_packages
import os
ldesc = open(os.path.join(os.path.dirname(__file__), 'README')).read()
setup(
name='cc-common',
version='8',
description='CloudControl common libraries',
long_description=ldesc,
author='Antoine Millet',
author_email='antoine.millet@smartjog.com',
url='https://intranet.fr.smartjog.net/forge/p/cc-common/',
license='LGPL3',
packages=find_packages(),
namespace_packages=['cloudcontrol'],
classifiers=[
'Operating System :: Unix',
'Programming Language :: Python',
],
)