diff --git a/ccserver/ccserver.py b/ccserver/ccserver.py index e32749c3d8e701cf20dc0093e663eeba0ed1ae09..4b75516f0aede6c62e0e664e73a3ae87b2c45bbd 100644 --- a/ccserver/ccserver.py +++ b/ccserver/ccserver.py @@ -8,8 +8,8 @@ Main class of cc-server. from __future__ import absolute_import import logging -from copy import copy from fnmatch import fnmatch as glob +from functools import partial from sjrpc.server import SSLRpcServer @@ -17,11 +17,13 @@ from ccserver.handlers import WelcomeHandler from ccserver.conf import CCConf from ccserver.client import CCClient from ccserver.exceptions import AlreadyRegistered, NotConnectedAccountError -from ccserver.orderedset import OrderedSet -from ccserver.tql import TqlParser, TqlObject -from ccserver.objectsdb import ObjectsDB from ccserver.jobs import JobsManager +from cloudcontrol.common.tql.db.object import TqlObject +from cloudcontrol.common.tql.db.tag import StaticTag, CallbackTag +from cloudcontrol.common.tql.db.db import TqlDatabase + + class CCServer(object): ''' CloudControl server main class. @@ -55,8 +57,7 @@ class CCServer(object): if keyfile: logging.info('SSL Key: %s', certfile) - - self.objects = ObjectsDB(self) + self.db = TqlDatabase() # Create the rpc server: logging.info('Listening on %s:%s', address, port) @@ -74,8 +75,7 @@ class CCServer(object): Update the database with accounts. ''' - all_objects = self.objects.all((), ()) - db_accounts = set([o['a'] for o in all_objects if 'a' in o]) + db_accounts = set((obj['a'].get_value() for obj in self.db.objects if 'a' in obj)) accounts = set(self.conf.list_accounts()) to_register = accounts - db_accounts @@ -83,11 +83,13 @@ class CCServer(object): for login in to_register: conf = self.conf.show(login) - obj = TqlObject(id=login, r=conf['role'], a=login) - self.objects.register(obj) + obj = TqlObject(login) + obj.register(StaticTag('r', conf['role'])) + obj.register(StaticTag('a', login)) + self.db.register(obj) for login in to_unregister: - self.objects.unregister(login) + self.db.unregister(login) def iter_connected_role(self, role=None): ''' @@ -119,9 +121,16 @@ class CCServer(object): # Create the object on objectdb if required: if create_object: - obj = TqlObject(id=login, r=role) - logging.info('%s' % obj) - self.objects.register(obj) + obj = TqlObject(login) + obj.register(StaticTag('r', role)) + self.db.register(obj) + else: + obj = self.db.get(login) + assert obj is not None + + # Define server defined tags for the new node: + obj.register(CallbackTag('con', self._connected[login].get_uptime, ttl=0)) + obj.register(CallbackTag('ip', self._connected[login].get_ip)) def unregister(self, connection): ''' @@ -129,18 +138,20 @@ class CCServer(object): :param connection: the connection of the client to unregister ''' - + #XXX: not functional since new db!!! client = self.search_client_by_connection(connection) # Unregister objects from database if it have no account attached: - obj = self.objects.get(client.login) + obj = self.db.get(client.login) if obj is not None and 'a' not in obj: - self.objects.unregister(obj['id']) + self.db.unregister(obj) + else: + # Unregister tags of connected account: + obj.unregister() if client.login in self._connected: del self._connected[client.login] - self.objects.unregister_children(client.login) - self.objects.clean_ttls(client.login) + #self.objects.unregister_children(client.login) def sub_register(self, parent, name, role): ''' @@ -151,10 +162,15 @@ class CCServer(object): :param role: the role of the subnode ''' - obj_parent = self.objects.get_by_id(parent) - oid = '%s.%s' % (parent, name) - obj = TqlObject(id=oid, r=role, __parent=obj_parent, p=obj_parent['id']) - self.objects.register(obj) + client = self.get_connection(parent) + child = '%s.%s' % (parent, name) + client.register_child(child) + + # Register the children in the tags database: + obj = TqlObject(child) + obj.register(StaticTag('r', role)) + obj.register(StaticTag('p', client)) + self.db.register(obj) def sub_unregister(self, parent, name): ''' @@ -164,8 +180,12 @@ class CCServer(object): :param login: the name of the subnode ''' - oid = '%s.%s' % (parent, name) - self.objects.unregister(oid) + client = self.get_connection(parent) + child = '%s.%s' % (parent, name) + client.unregister_child(child) + + # Unregister the children from the tags database: + self.objects.unregister(child) def search_client_by_connection(self, connection): ''' @@ -234,77 +254,75 @@ class CCServer(object): rights = self.conf.show(login)['rights'] if tql is not None: - objects = self.list(tql, pure=True) + objects = self.db.raw_query(tql) for right in rights: if not (right['method'] is None or glob(method, right['method'])): continue if tql is not None and right['tql']: - objects_right = self.list(right['tql'], pure=True) + objects_right = self.db.raw_query(right['tql']) if set(objects) <= set(objects_right): return right['target'] == 'allow' else: return right['target'] == 'allow' return False - def list(self, query, show=set(), pure=False, return_toshow=False): + def tags_register(self, login, name, ttl=None, value=None): ''' - List objects on the server. + Register a new tag for a client. - :param query: the TQL to use to selection objects on list. + :param login: login of the client + :param name: name of the tag to register + :param ttl: TTL of the tag if applicable (None = no TTL, the tag will + never expire) + :param value: value of the tag ''' - self._update_accounts() + obj = self.db.get(login) + client = self._connected.get(login) + callback = partial(client.get_remote_tags, name) + tag = CallbackTag(name, callback, ttl=ttl) + obj.register(tag) - parser = TqlParser(query) - ast, to_show, to_get, to_check = parser.parse() - - orig_to_show = copy(to_show) - to_show += show - - # Calculate the tags to get/check/show: - if to_get is not None: - to_get -= set((self.RESERVED_TAGS)) - if to_check is not None: - to_check -= set((self.RESERVED_TAGS)) - - deny = set() - - for tag in copy(to_show): - if tag == '*': - to_show = None - deny.clear() - elif tag.startswith('-'): - tag = tag[1:] - deny.add(tag) - if to_show is not None and tag in to_show: - to_show.remove(tag) - - if to_show is None: - to_display = None - else: - to_display = set(to_show) | to_get + def tags_unregister(self, login, name): + ''' + Unregister a tag for the client. - if to_show is not None: - to_show = set(to_show) - to_show -= set((self.RESERVED_TAGS)) + :param login: login of the client + :param name: name of the tag to unregister + ''' - objects = OrderedSet(self.objects.all(to_get, to_check)) - if ast is not None: - objects, _ = ast.eval(objects, objects) + obj = self.db.get(login) + obj.unregister(name) - ids = [x['id'] for x in objects] + def tags_drop(self, login, name): + ''' + Drop the cached value of a tag for the client. - objects = self.objects.some(ids, to_show) + :param login: login of the client + :param name: name of the tag to drop + ''' + obj = self.db.get(login) + tag = obj.get(name) + if tag is not None: + tag.invalidate() - if pure: - return objects - - objects_dicts = [] - - for obj in objects: - objects_dicts.append(obj.to_dict(to_display, deny=deny)) + def tags_update(self, login, name, value=None, ttl=None): + ''' + Update a tag. - if return_toshow: - return objects_dicts, orig_to_show - else: - return objects_dicts + :param login: login of the client + :param name: name of the tag to update + :param value: new value of the tag + :param ttl: new ttl of the tag + ''' + obj = self.db.get(login) + tag = obj.get(name) + if tag is not None: + if value is not None: + tag.set_value(value) + if ttl is not None: + tag.ttl = ttl + + def list(self, query, show=None): + self._update_accounts() + return self.db.query(query, show) diff --git a/ccserver/client.py b/ccserver/client.py index b6446acd52b1340bb2fc70ea451c38c2bb6d266e..45af66d08ef76e6d8dcb589ac57be36217e44d18 100644 --- a/ccserver/client.py +++ b/ccserver/client.py @@ -88,3 +88,6 @@ class CCClient(object): tags['con'] = self.get_uptime() tags['ip'] = self.get_ip() return tags + + def get_remote_tags(self, tag): + return self.connection.call('get_tags', (tag,))[tag] diff --git a/ccserver/handlers.py b/ccserver/handlers.py index fff28dec74bd9b1732467c10926543bd91006c21..c4992713d5b36a1fd75189441bd1579bbdfc657f 100644 --- a/ccserver/handlers.py +++ b/ccserver/handlers.py @@ -175,6 +175,54 @@ class HypervisorHandler(OnlineCCHandler): client = self._server.search_client_by_connection(conn) self._server.sub_unregister(client.login, obj_id) + @listed + @pass_connection + def tags_register(self, conn, name, ttl=None, value=None): + ''' + Register a new tag on the calling node. + + :param name: name of the tag to register + :param ttl: ttl of the tag (or None if not applicable) + :param value: value to fill the tag (optionnal) + ''' + client = self._server.search_client_by_connection(conn) + self._server.tags_register(client.login, name, ttl, value) + + @listed + @pass_connection + def tags_unregister(self, conn, name): + ''' + Unregister a tag on the calling node. + + :param name: name of the tag to unregister + ''' + client = self._server.search_client_by_connection(conn) + self._server.tags_unregister(client.login, name) + + @listed + @pass_connection + def tags_drop(self, conn, name): + ''' + Drop the tag value of the specified tag on the calling node. + + :param name: name of the tag to drop + ''' + client = self._server.search_client_by_connection(conn) + self._server.tags_drop(client.login, name) + + + @listed + @pass_connection + def tags_update(self, conn, name, value, ttl=None): + ''' + Update the value of the specified tag on the calling node. + + :param name: name of the tag to update + :param value: new tag value + :param ttl: new ttl value + ''' + client = self._server.search_client_by_connection(conn) + self._server.tags_update(client.login, name, value, ttl) class CliHandler(OnlineCCHandler): ''' @@ -222,12 +270,12 @@ class CliHandler(OnlineCCHandler): :param query: the query to select objects to show ''' - self._check(conn, 'list', query) + #self._check(conn, 'list', query) logging.debug('Executed list function with query %s', query) - objects, tags = self._server.list(query, return_toshow=True) + objects = self._server.list(query) order = OrderedSet(['id']) - if tags is not None: - order |= OrderedSet(tags) + #if tags is not None: + # order |= OrderedSet(tags) return {'objects': objects, 'order': list(order)} def _vm_action(self, query, method, *args, **kwargs): @@ -669,7 +717,7 @@ class CliHandler(OnlineCCHandler): ''' self._check(conn, 'execute', query) - objects = self._server.list(query, show=set(('r',))) + objects = self._server.list(query, show=('r',)) errs = Reporter() for obj in objects: if obj['r'] not in ('hv', 'host'): @@ -856,13 +904,6 @@ class CliHandler(OnlineCCHandler): 'hv_dest': dest['id'], 'author': client.login}) - @listed - def dbstats(self): - ''' - Return statistics about current database status. - ''' - return self._server.objects.stats() - class BootstrapHandler(OnlineCCHandler): diff --git a/ccserver/objectsdb.py b/ccserver/objectsdb.py deleted file mode 100644 index ddd07bd7bbd732ca12d696bba56905ffb9bd8173..0000000000000000000000000000000000000000 --- a/ccserver/objectsdb.py +++ /dev/null @@ -1,437 +0,0 @@ -#!/usr/bin/env python -#coding=utf8 - -''' -The current living objects database. -''' - -from __future__ import absolute_import - -from datetime import datetime, timedelta -import logging -from threading import RLock - -from sjrpc.core.async import AsyncWatcher - -from ccserver.tql import TqlObject -from ccserver.orderedset import OrderedSet -from ccserver.exceptions import AlreadyRegistered, UnknownObjectError - -DEFAULT_TTL = 0 -TTL_SERVER_DELTA = 1 # Delta to apply to TTL for all tags - -class ObjectsDB(object): - ''' - The object database. - - This class abstract the access to the current living objects database. The - database store a list of :class:`TqlObject` instances, each defining an - object with its tags. - - Objects are registered by :meth:`register` method, and removed by - :meth:`unregister` method. You can update the tags of registered objects - with :meth:`update` method. A call to :meth:`all` method give you the - complete list of objects with an automatic update before to return it. - - .. note: - All calls to the :class:`ObjectDB` are thread-safe. - ''' - - def __init__(self, server): - - # The server to use to complete database: - self._server = server - - # The object database: - self._objects = {} # Dict of oid -> TqlObject instances - - # The TTL database: - self._ttls = {} # (obj_id, tag) -> time of death - - # The access lock of the object database: - self._lock = RLock() - - self._requested = {} # object id -> set of tags - # Asynchronous watcher: - self._asyncwatcher = None - - # Total amount of query updates: - self._nb_queries = 0 - - def update(self, ids=None, tags=None, tags_novalue=None): - ''' - Update the database according to the TTL values. - - :param ids: list of ids to update (or None for all) - :param tags: list of tags to update (or None for all) - :param tags_noval: list of tags to update (only the name, not the - attached value, None for all) - ''' - - with self._lock: - self._requested = {} - self._nb_queries += 1 - self._update(ids, tags, tags_novalue) - self._process_responses(tags) - self._update_user_defined(ids) - - def _update(self, ids=None, tags=None, tags_novalue=None): - ''' - First step of the database update process. - - This methods make essentially two things: - - * Calculate the final list of tags to get and to check - * Send the request to each client involved in the update - - .. warning: - The call to this method is not threadsafe and must be done only - by :meth:`update` method. - ''' - - self._asyncwatcher = AsyncWatcher() - - # Cast input to the right type: - if tags is not None: - tags = set(tags) - - if tags_novalue is not None: - tags_novalue = set(tags_novalue) - if tags is not None: - tags_novalue -= tags - - if ids is not None: - ids = set(ids) - else: - ids = set(self._objects) - - # The main-loop, we will pop each object id from the set and - # process it: - while ids: - oid = ids.pop() - obj = self._objects.get(oid) - - if obj is None: - raise UnknownObjectError('%r not found in database' % oid) - - if '__parent' in obj: - # Current object is proxified by a parent: - parent = obj['__parent'] - - # Parent is removed from the list if it present (because - # it has been updated): - if parent['id'] in ids: - self._update_object(parent, tags, tags_novalue) - ids.remove(parent['id']) - else: - self._update_object(parent, (), (), only_udt=True) - - # Search for sibling objects: - sg = set((o for o in ids if o.startswith('%s.' % parent['id']))) - sg &= ids - ids -= sg - sg.add(oid) - - for sibling in sg: - sibling = self.get_by_id(sibling) - self._update_object(sibling, tags, tags_novalue) - else: - # Current object is a standard node: - self._update_object(obj, tags, tags_novalue) - - def _update_user_defined(self, ids): - ''' - Update the user defined tags (previously fetched - by :meth:`_update_object`). - ''' - - if ids is not None: - ids = set(ids) - else: - ids = set(self._objects) - - for objid in ids: - obj = self.get_by_id(objid) - - if '__parent' in obj: - user_tags = obj['__parent']['__user_defined_tags'] - else: - user_tags = obj['__user_defined_tags'] - - obj.update(user_tags) - - # Delete ttls if present: - for tag in user_tags: - if (obj['id'], tag) in self._ttls: - del self._ttls[(obj['id'], tag)] - - def _process_responses(self, tags): - ''' - Process the responses received from clients. - ''' - - - # Avoid error if all tags are asked (not optimized case because we - # can't known the real list of tags: - if tags is None: - tags = () - - received = set() - - now = datetime.now() - for msg in self._asyncwatcher.iter(timeout=5): - obj = msg['data'] - assert isinstance(obj, TqlObject), 'data obj is not TqlObject' - - # Update tags if received from the peer: - if msg.get('error') is not None: - logging.warning('Error from client while getting tags: %s', - msg.get('error')) - for name in tags: - if name in obj: - old = obj[name] - obj[name] = '#ERR#OLD:%s' % old - else: - obj[name] = '#ERR#' - self._ttls[(obj['id'], name)] = now + timedelta(0, 10) - else: - returned = msg['return'] - assert isinstance(returned, dict), 'returned tags is not a dict' - received.add(obj['id']) - - for name, attrs in returned.iteritems(): - obj[name] = attrs.get('value') - - # Update the ttl - ttl = attrs.get('ttl', DEFAULT_TTL) - if ttl == -1: - tod = datetime.max - else: - ttl += TTL_SERVER_DELTA - dt = timedelta(seconds=ttl) - tod = now + dt - - self._ttls[(obj['id'], name)] = tod - - for oid in set(self._requested) - received: - obj = self.get_by_id(oid) - o_tags = self._requested[oid] - if o_tags is not None: - for name in o_tags: - obj[name] = '#ERR#' - self._ttls[(obj['id'], name)] = now + timedelta(0, 10) - - def _update_object(self, obj, tags, tags_novalue, only_udt=False): - ''' - Calculate the list of tags to request for a client, and send the - request to the client. - - :param obj: the object to update - :param tags: the tags to update on the object - :param tags_novalue: the tags to update (do not fetch values) on the - object - :param only_udt: only update the user defined tags - ''' - - parent = obj.get('__parent') - - if parent is None: - if 'a' in obj: - user_tags = self._server.conf.show(obj['id'])['tags'] - obj['__user_defined_tags'] = user_tags - else: - user_tags = {} - obj['__user_defined_tags'] = {} - client_id = obj['id'] - else: - user_tags = parent['__user_defined_tags'] - client_id = parent['id'] - - if only_udt: - return - - if tags is not None: - tags = set(tags) - tags -= set(user_tags) - - if tags_novalue is not None: - tags_novalue = set(tags_novalue) - tags_novalue -= set(user_tags) - - try: - client = self._server.get_connection(client_id) - except KeyError: - # The client is not connected, we need to keep only unttlised tags: - for name in obj.keys(): - if (obj['id'], name) in self._ttls: - del obj[name] - else: - # The client is connected: - if parent is None: - serv_tags = client.get_tags() - obj.update(serv_tags) - # Add an infinite ttl to server defined tags: - for tag in serv_tags: - self._ttls[(obj['id'], tag)] = datetime.max - - # First, check if each tag is not already cached: - now = datetime.now() - if tags is not None: - for tag in tags.copy(): - if now < self._ttls.get((obj['id'], tag), now): - tags.remove(tag) - # Cast as jsonizable type: - tags = tuple(tags) - - if tags_novalue is not None: - for tag in tags_novalue.copy(): - if now < self._ttls.get((obj['id'], tag), now): - tags_novalue.remove(tag) - # Cast as jsonizable type: - tags_novalue = tuple(tags_novalue) - - # Ask tags to the node: - if (tags is None or tags) or (tags_novalue is None or tags_novalue): - self._request_tags(obj, client, tags, tags_novalue) - - def _request_tags(self, obj, client, tags, tags_novalue): - ''' - Send the request to the client. - ''' - - if '__parent' in obj: - subid = obj['id'].partition('.')[2] - self._asyncwatcher.register(client.connection, 'sub_tags', subid, - tags, tags_novalue, _data=obj) - else: - self._asyncwatcher.register(client.connection, 'get_tags', - tags, tags_novalue, _data=obj) - - self._requested[obj['id']] = tags - - def get_by_id(self, oid): - ''' - Get an registered object by its id. - - :param oid: the complete oid of the object to get - :return: a :class:`TqlObject` instance - ''' - return self._objects[oid] - - def register(self, obj): - ''' - Populate the database with the specified object. All the tags specified - in the passed object are not declared in cache and will never be - cleaned. - - :param obj: the object to register - ''' - - with self._lock: - if 'id' not in obj: - raise ValueError('id tag must exists on the object') - elif obj['id'] in self._objects: - raise AlreadyRegistered('The object is already registered') - else: - self._objects[obj['id']] = obj - - def unregister(self, obj_id): - ''' - Remove an object from the manager by its id. - ''' - - with self._lock: - if obj_id in self._objects: - self.unregister_children(obj_id) - self.clean_ttls(obj_id) - del self._objects[obj_id] - else: - raise ValueError('The object %r is not registered' % obj_id) - - def unregister_children(self, obj_id): - ''' - Remove all children of an object. - ''' - - with self._lock: - if obj_id in self._objects: - parent = self._objects[obj_id] - - for key, obj in self._objects.items(): - if obj.get('__parent') is parent: - self.clean_ttls(obj['id']) - del self._objects[key] - else: - raise ValueError('The object %r is not registered' % obj_id) - - def clean_ttls(self, obj_id): - ''' - Remove TTLs for all tags for the specified object. - ''' - - with self._lock: - obj = self.get_by_id(obj_id) - for oid, tag in self._ttls.keys(): - if oid == obj_id: - del self._ttls[(oid, tag)] - try: - del obj[tag] - except: - pass - - def get_ids(self): - ''' - Get the set of all the first level ids. - ''' - - ids = set() - for obj in self._objects.values(): - if not '__parent' in obj: - ids.add(obj['id']) - return ids - - def get(self, oid): - ''' - Get the specified object by it id or return None. - ''' - return self._objects.get(oid) - - def all(self, tags, to_check): - ''' - Get all the objects registered on the server with specified tags. - ''' - - self.update(tags=tags, tags_novalue=to_check) - for obj in self._objects.values(): - yield obj - - def some(self, ids, tags=set(), to_check=set()): - ''' - Get specified objects registed on the server with specified tags. - ''' - - self.update(ids, tags=tags, tags_novalue=to_check) - for obj_id in ids: - yield self._objects[obj_id] - - def stats(self): - ''' - Get stats about the current state of the object database. - ''' - - stats = {} - - stats['count'] = len(self._objects) - cached = dead = 0 - now = datetime.now() - for date in self._ttls.values(): - if date > now: - dead += 1 - else: - cached += 1 - stats['cached'] = cached - stats['dead'] = dead - stats['total'] = cached + dead - stats['queries'] = self._nb_queries - - return stats diff --git a/ccserver/tql.py b/ccserver/tql.py deleted file mode 100644 index 194cfd32ceaaea0dc049d0d6bc7e469e4377734c..0000000000000000000000000000000000000000 --- a/ccserver/tql.py +++ /dev/null @@ -1,776 +0,0 @@ -#!/usr/bin/env python -#coding=utf8 - -''' - Tag Query Language tools module. -''' - -from __future__ import absolute_import - -import re -from fnmatch import fnmatch -from ccserver.orderedset import OrderedSet - -# Helpers functions: - -PREFIXES = {'b': 1, 'o': 1, - 'k': 1000, 'ki': 1024, - 'm': 1000**2, 'mi': 1024**2, - 'g': 1000**3, 'gi': 1024**3, - 't': 1000**4, 'ti': 1024**4, - 'p': 1000**5, 'pi': 1024**5} - -def prefix(number): - ''' - Cast as integer a prefixed string (eg: "42M" -> 42000000) - ''' - number = str(number).lower() - for sym, mul in PREFIXES.items(): - if number.endswith(sym): - num = number.rstrip(sym) - try: - return float(num) * mul - except ValueError: - return number - else: - try: - return float(number) - except ValueError: - return number - -# End of helpers. - -class TqlParsingError(Exception): - ''' - Error raised when a parsing error is occured. - ''' - pass - - -class TqlObject(dict): - ''' - A special dict used to represent a TQL object. Each key/value of the - :class:`TqlObject` instance is a tag associated to its value. - - It differs from a classical dict by this points: - * An 'id' key *MUST* exist and be passed to the constructor, this key - is also read-only and can't be removed. - * The :meth:`__hash__` method is defined and allow to store - :class:`TqlObjects` instance in sets. - * An :meth:`to_dict` method allow to export the object as a standard dict. - ''' - - def __init__(self, *args, **kwargs): - super(TqlObject, self).__init__(*args, **kwargs) - if 'id' not in self: - raise ValueError('"id" key must be defined in constructor') - - def __hash__(self): - return self['id'].__hash__() - - def __setitem__(self, key, value): - if key == 'id': - raise KeyError('Key %r in read-only.' % key) - else: - super(TqlObject, self).__setitem__(key, value) - - def __delitem__(self, key): - if key == 'id': - raise KeyError('Key %r in read-only.' % key) - else: - super(TqlObject, self).__delitem__(key) - - def to_dict(self, tags, deny=None): - ''' - Export the :class:`TqlObject` as standard Python dictionnary. This - method takes two arguments used to define the set of tags to include in - the exported dictionnary. - - :param tags: the list of tags to export or None if you want to export - all tags - :param deny: useful in case where tags is None, to exclude some tags - from the export - - .. note: - This method doesn't export private tags (tags which starts with '_') - ''' - - if tags is not None: - tags = set(tags) - tags.add('id') - - if deny is None: - deny = set() - else: - deny = set(deny) - deny -= set(('id',)) - - exported = {} - - for key, value in self.iteritems(): - if not key.startswith('_'): - if (key not in deny and (tags is None or key in tags) - and value is not None): - exported[key] = str(value) - - return exported - - -class TqlLexer(object): - - ''' - Simple tokenizer for the TQL language. - - :param tql: the TQL string to tokenize. - ''' - - TOK_EOL = 1 - TOK_WORD = 2 - TOK_SEP = 3 - TOK_OP = 4 - TOK_PAR_OPEN = 5 - TOK_PAR_CLOSE = 6 - - CHAR_QUOTES = '"\'' - CHAR_SEP = '&|^$%' - CHAR_OP = '<>=:!~' - CHAR_PAR_OPEN = '(' - CHAR_PAR_CLOSE = ')' - - def __init__(self, tql): - # The string representing the query to tokenize: - self._buf = tql - - # The current position of the cursor in the string: - self._cursor = 0 - - # The current processed token cache: - self._tokens_cache = [] - - def get_token(self): - ''' - Get the next token of the query and forward. - ''' - - if not self._tokens_cache: - # If the cache is empty, get the next token and return it directly: - return self._get_next_token() - else: - # If the cache is not empty, return its first token: - return self._tokens_cache.pop(0) - - def look_ahead(self, upto=1): - ''' - Get the next token of the query but don't forward (multiple look_ahead - calls will return the same token of get_token is not called). - ''' - - if len(self._tokens_cache) < upto: - # The cache is not populated with enough tokens, we process the - # difference between the look ahead value and the current number - # of cached tokens available: - for _ in xrange(upto - len(self._tokens_cache)): - self._tokens_cache.append(self._get_next_token()) - - # And return the cached token: - return self._tokens_cache[upto - 1] - - def _get_next_token(self): - ''' - Process the next token and return it. - ''' - - # The buffer which store the value of current token: - ctoken = '' - - # The type of the current processed token: - ctype = None - - # The current opened quote and it escapment flag: - cquote = None - cescape = False - - while self._cursor < len(self._buf): - cur = self._buf[self._cursor] - - if cquote: - # We are in "quote mode", we will eat each characted until - # the closing quote is found: - self._cursor += 1 - if cur == cquote and not cescape: - # Closing quote is reached, we can return - # the word token: - return (self.TOK_WORD, ctoken) - elif cur == '\\': # Escapement handling - cescape = True - else: - ctoken += cur - cescape = False - elif cur in self.CHAR_SEP: - # The current character is a separator, two cases: - if ctoken: - # The current token buffer is not empty, we have reached - # the end of the current token, so we return it but we - # don't forward the cursor: - return (ctype, ctoken) - else: - # The current token buffer is empty, we process this new - # separator token and return it (we directly return it since - # separator tokens are composed of a single character): - self._cursor += 1 - return (self.TOK_SEP, cur) - elif cur in self.CHAR_OP: - # The current character is a part of an operator: - if ctype is not None and ctype != self.TOK_OP: - # If we just entered in a new operator token, we return the - # last processed token without forwarding the cursor: - return (ctype, ctoken) - else: - # Else, we process this new operator token and forward the - # cursor: - ctype = self.TOK_OP - ctoken += cur - self._cursor += 1 - elif cur in self.CHAR_QUOTES: - # The current character is an opening quote, we enter in a mode - # where all characters of the buffer will be eaten until the - # closing quote is reached: - if ctype is not None: - return (ctype, ctoken) - cquote = cur - self._cursor += 1 - elif cur == self.CHAR_PAR_OPEN: - # The current caracter is an opening parenthesis: - if ctoken: - # The current token buffer is not empty, we have reached - # the end of the current token, so we return it but we - # don't forward the cursor: - return (ctype, ctoken) - else: - self._cursor += 1 - return (self.TOK_PAR_OPEN, '(') - elif cur == self.CHAR_PAR_CLOSE: - # The current caracter is an opening parenthesis: - if ctoken: - # The current token buffer is not empty, we have reached - # the end of the current token, so we return it but we - # don't forward the cursor: - return (ctype, ctoken) - else: - self._cursor += 1 - return (self.TOK_PAR_CLOSE, ')') - else: - # The current character is a part of a word token - if ctype is not None and ctype != self.TOK_WORD: - # If we just entered in a new word token, we return the - # last processed token without forwarding the cursor: - return (ctype, ctoken) - else: - # Else, we process this new word token and forward the - # cursor: - ctype = self.TOK_WORD - ctoken += cur - self._cursor += 1 - - # We have reached the end of the buffer: - if cquote: - # We are in "quote mode", we need to raise a parsing error - # because the quote is not closed: - raise TqlParsingError('Quote not closed') - elif ctoken: - # The current token buffer is not empty, we return this - # current token: - return (ctype, ctoken) - else: - # The current token buffer is empty, we return the EOL - # token: - return (self.TOK_EOL, 'eol') - - -class TqlAst(object): - ''' - Base class for all AST components. - ''' - pass - - -class TqlAstAll(object): - ''' - Represents the complete set of objects. - ''' - - def __init__(self): - pass - - def eval(self, objects, all_objects): - return OrderedSet(all_objects), all_objects - - -class TqlAstTag(TqlAst): - ''' - A single tag in the TQL AST. Tag name can be prefixed of a "-" - character to invert selection behavior. - ''' - - def __init__(self, name): - if name.startswith('-'): - self.negate = True - name = name[1:] - else: - self.negate = False - self.name = name - - def get_name(self): - return '-' + self.name if self.negate else self.name - - def to_dot(self): - ''' - Return the DOT format representation of the current AST node and - sub-nodes. - ''' - - stmts = [] - name = id(self) - stmts.append('%s [label="%s"];' % (name, self.name)) - return stmts - - def eval(self, objects, all_objects): - objs = OrderedSet() - for obj in objects: - if (not self.negate and self.name in obj - or self.negate and not self.name in obj): - objs.add(obj) - return objs, all_objects - - -class TqlAstTagCondition(TqlAst): - ''' - A tag attached to a condition (operator + value) which select only - the items which match the condition. - ''' - - OPERATORS = {':': 'op_glob', - '=': 'op_equal', - '>': 'op_gt', - '<': 'op_lt', - '>=': 'op_gte', - '<=': 'op_lte', - '~': 'op_regex'} - - NEGATABLE_OPERATORS = (':', '=', '~') - - def __init__(self, name, operator, value): - self.name = name - self.operator = operator - self.value = value - - if self.operator.startswith('!'): - negate = True - self.operator = self.operator[1:] - if self.operator not in self.NEGATABLE_OPERATORS: - raise TqlParsingError('Bad operator %r' % operator) - else: - negate = False - - op_funcname = self.OPERATORS.get(self.operator) - if op_funcname is None: - raise TqlParsingError('Bad operator %r' % self.operator) - else: - if hasattr(self, op_funcname): - func = getattr(self, op_funcname) - if negate: - self.operator_func = lambda l, r: not func(l, r) - else: - self.operator_func = func - else: - raise TqlParsingError('Operator function not found ' - '%r' % op_funcname) - - def to_dot(self): - ''' - Return the DOT format representation of the current AST node and - sub-nodes. - ''' - - stmts = [] - name = id(self) - stmts.append('%s [label="%s %s %s"];' % (name, self.name, self.operator, - self.value)) - return stmts - - def eval(self, objects, all_objects): - objs = OrderedSet() - for obj in objects: - if self.name in obj: - value = obj[self.name] - if self.operator_func(value, self.value): - objs.add(obj) - return objs, all_objects - - def op_glob(self, value, pattern): - ''' - The globbing operator, handle * and ? characters. - ''' - - if value is None: - return False - return fnmatch(str(value), str(pattern)) - - def op_lt(self, lvalue, rvalue): - ''' - The lesser than operator. - ''' - - lvalue = prefix(lvalue) - rvalue = prefix(rvalue) - return lvalue < rvalue - - def op_lte(self, lvalue, rvalue): - ''' - The lesser or equal than operator. - ''' - - lvalue = prefix(lvalue) - rvalue = prefix(rvalue) - return lvalue <= rvalue - - def op_gt(self, lvalue, rvalue): - ''' - The greater than operator. - ''' - - lvalue = prefix(lvalue) - rvalue = prefix(rvalue) - return lvalue > rvalue - - def op_gte(self, lvalue, rvalue): - ''' - The greater or equal than operator. - ''' - - lvalue = prefix(lvalue) - rvalue = prefix(rvalue) - return lvalue >= rvalue - - def op_regex(self, value, pattern): - ''' - The regular expression operator. - ''' - - if value is None: - return False - try: - return re.match(pattern, value) is not None - except re.error: - raise TqlParsingError('Error in your regex pattern: %s' % pattern) - - def op_equal(self, lvalue, rvalue): - ''' - The strict equal operator. - ''' - - lvalue, rvalue = str(lvalue), str(rvalue) - if lvalue is None: - return False - if lvalue.isdigit() and rvalue.isdigit(): # Integer comparison: - lvalue = prefix(lvalue) - rvalue = prefix(rvalue) - - return lvalue == rvalue - - -class TqlAstBinarySeparators(TqlAst): - ''' - Base class for binary separators. - ''' - - node_token = None - - def __init__(self, left, right): - self.left = left - self.right = right - - def to_dot(self): - ''' - Return the DOT format representation of the current AST node and - sub-nodes. - ''' - - stmts = [] - name = id(self) - stmts.append('%s [shape=circle, label="%s"];' % (name, self.node_token)) - stmts.append('%s -> %s [label="l"];' % (name, id(self.left))) - stmts.append('%s -> %s [label="r"];' % (name, id(self.right))) - stmts += self.left.to_dot() - stmts += self.right.to_dot() - return stmts - - def eval(self, objects, objects_all): - raise NotImplementedError('This function must be implemented ' - 'in derivated classes') - - -class TqlAstSorter(TqlAst): - ''' - A sorting separator. - ''' - - node_token = '%' - - def __init__(self, child, name): - self.name = name - self.child = child - - def to_dot(self): - ''' - Return the DOT format representation of the current AST node and - sub-nodes. - ''' - - stmts = [] - name = id(self) - stmts.append('%s [label="%s (%s)"];' % (name, self.node_token, - self.name.name)) - stmts.append('%s -> %s;' % (name, id(self.child))) - stmts += self.child.to_dot() - return stmts - - def eval(self, objects, all_objects): - objects, _ = self.child.eval(objects, all_objects) - sorted_objects = sorted(objects, key=self._sort_getter) - if self.name.negate: - sorted_objects = reversed(sorted_objects) - return sorted_objects, all_objects - - def _sort_getter(self, obj): - value = obj.get(self.name.name) - - try: - value = float(str(value)) - except ValueError: - pass - - return value - -class TqlAstLimit(TqlAst): - ''' - A limitation separator. - ''' - - node_token = '^' - - def __init__(self, child, obj_slice): - length, _, offset = obj_slice.partition(',') - if offset: - self.start = int(offset) - else: - self.start = 0 - if length: - self.stop = self.start + int(length) - else: - self.stop = None - self.child = child - - def to_dot(self): - ''' - Return the DOT format representation of the current AST node and - sub-nodes. - ''' - - stmts = [] - name = id(self) - stmts.append('%s [label="%s (%s,%s)"];' % (name, self.node_token, - self.start, self.stop)) - stmts.append('%s -> %s;' % (name, id(self.child))) - stmts += self.child.to_dot() - return stmts - - def eval(self, objects, all_objects): - objects, _ = self.child.eval(objects, all_objects) - return tuple(objects)[self.start:self.stop], all_objects - - -class TqlAstIntersect(TqlAstBinarySeparators): - ''' - An intersection separator. - ''' - - node_token = '&' - - def eval(self, objects, all_objects): - objects = OrderedSet(objects) - left = OrderedSet(self.left.eval(objects, all_objects)[0]) - right = OrderedSet(self.right.eval(objects, all_objects)[0]) - return left & right, all_objects - - -class TqlAstUnion(TqlAstBinarySeparators): - ''' - An union separator. - ''' - - node_token = '|' - - def eval(self, objects, all_objects): - objects = OrderedSet(objects) - left = OrderedSet(self.left.eval(objects, all_objects)[0]) - right = OrderedSet(self.right.eval(objects, all_objects)[0]) - return left | right, all_objects - - -class TqlParser(object): - ''' - Parse a TQL query and return the AST and the list of tags to show. - ''' - - def __init__(self, query): - self._query = query - self._lexer = TqlLexer(query) - self._to_show = [] - self._to_get = set() - self._to_check = set() - - def parse(self): - ''' - Parse and return the AST of the TQL query. This method also returns - the list of tags to show, to get and to check. - ''' - - self._to_show = [] - self._to_get.clear() - self._to_check.clear() - return (self._parse(), self._to_show, self._to_get, self._to_check) - - def _parse(self): - # Watch the next token to process: - token = self._lexer.look_ahead() - - # Token is a word, we will parse it: - if token[0] == self._lexer.TOK_WORD: - word = self._parse_expression() - token = self._lexer.look_ahead() - - # The common token after a name is a separator: - if token[0] == self._lexer.TOK_SEP: - return self._parse_separator(word) - elif token[0] in (self._lexer.TOK_EOL, self._lexer.TOK_PAR_CLOSE): - return word - else: - raise TqlParsingError('Unexpected token %s' % token[1]) - - # Token is an opening parenthesis, we recursively call the _parse - # method to process the sub-query: - elif token[0] == self._lexer.TOK_PAR_OPEN: - self._lexer.get_token() - tree = self._parse() - token = self._lexer.look_ahead() - - # An sub-query must be closed by a closing parenthesis: - if token[0] == self._lexer.TOK_PAR_CLOSE: - self._lexer.get_token() - - # Check if sub-query is not followed by another sub-query: - token = self._lexer.look_ahead() - if token[0] == self._lexer.TOK_SEP: - return self._parse_separator(tree) - elif token[0] in (self._lexer.TOK_EOL, - self._lexer.TOK_PAR_CLOSE): - return tree - else: - raise TqlParsingError('Unexpected token %r' % token[1]) - else: - raise TqlParsingError('Unexpected token %r' % token[1]) - - elif token[0] == self._lexer.TOK_SEP: - return self._parse_separator(TqlAstAll()) - - # Token is the EOL: - elif token[0] == self._lexer.TOK_EOL: - return None - - else: - raise TqlParsingError('Parsing error, WORD expected.') - - def _parse_separator(self, left): - # Getting the operator token and the token to it right: - separator = self._lexer.get_token() - right = self._lexer.look_ahead() - - # Checking if separator token is really a separator: - if separator[0] != self._lexer.TOK_SEP: - raise TqlParsingError('Unexpected token %r' % separator[1]) - - # Checking if the token to the right of the separator is an opening - # parenthesis or a word and parse it: - if right[0] == self._lexer.TOK_PAR_OPEN: - right = self._parse() - elif right[0] == self._lexer.TOK_WORD: - if separator[1] in '&|': - # The right token of a & or | separator can be a single - # token 'word', or an expression: - right = self._parse_expression() - elif separator[1] == '^': - # The right token of a ^ separator is not a tag but - # a slice (x,y): - right = right[1] - self._lexer.get_token() - elif separator[1] == '$': - # $ separator is not handler in AST: - right = self._lexer.get_token()[1] - else: - right = self._parse_word() - else: - raise TqlParsingError('Unexpected token %r' % separator[1]) - - # Create the AST node for each separator: - if separator[1] == '&': - tree = TqlAstIntersect(left, right) - elif separator[1] == '|': - tree = TqlAstUnion(left, right) - elif separator[1] == '^': - tree = TqlAstLimit(left, right) - elif separator[1] == '%': - tree = TqlAstSorter(left, right) - elif separator[1] == '$': - self._to_show.append(right) - tree = left - else: - raise TqlParsingError('Bad separator %r' % separator[1]) - - # Process the next token, maybe another separator, or the - # end of line / sub-query: - token = self._lexer.look_ahead() - if token[0] == self._lexer.TOK_SEP: - return self._parse_separator(tree) - elif token[0] in (self._lexer.TOK_EOL, self._lexer.TOK_PAR_CLOSE): - return tree - else: - raise TqlParsingError('Unexpected token %r' % token[1]) - - def _parse_word(self): - word = self._lexer.get_token() - - # Add the tag name to the list of tags to get: - self._to_check.add(word[1]) - self._to_show.append(word[1].lstrip('-')) - - return TqlAstTag(word[1]) - - def _parse_expression(self): - left = self._lexer.get_token() - next = self._lexer.look_ahead() - - # Add the tag name to the list of tags to get: - self._to_get.add(left[1].lstrip('-')) - self._to_show.append(left[1].lstrip('-')) - - if next[0] == self._lexer.TOK_OP: - operator = self._lexer.get_token() - right = self._lexer.get_token() - - if right[0] != self._lexer.TOK_WORD: - raise TqlParsingError('Unexpected token %r' % right[1]) - - return TqlAstTagCondition(left[1], operator[1], right[1]) - else: - return TqlAstTag(left[1])