Added TQL database (0fe45e1a) · Commits · Mirror / cc-common

cloudcontrol/common/tql/db/init.py

0 → 100644

+0 −0

Empty file added.

cloudcontrol/common/tql/db/db.py

0 → 100644

+351 −0

Original line number	Diff line number	Diff line
		""" Main class of the TQL database.
		"""

		from copy import copy
		from fnmatch import fnmatch

		import ply.yacc as yacc

		from cloudcontrol.common.tql.parser.parser import TqlParser
		from cloudcontrol.common.tql.db.object import TqlObject
		from cloudcontrol.common.tql.db.requestor import StaticRequestor
		from cloudcontrol.common.datastructures.ordereddict import OrderedDict
		from cloudcontrol.common.tql.parser.ast import (Filter, FilterPresence,
		UnionOperator,
		IntersectionOperator,
		ShowOperator, LimitOperator,
		SortOperator)

		#
		# Response objects:
		#


		class TqlResponseObject(object):

		""" A proxy for an object returned after a query.
		"""

		def __init__(self, object_to_wrap):
		self.object = object_to_wrap
		self._show = set(('id',))
		self._tags = {} # Mapping between fetched tags and its value

		def __repr__(self):
		return '<ResponseObject of %s>' % self.object

		#
		# Bind special methods:
		#

		def __contains__(self, value):
		return self.object.__contains__(value)

		def __hash__(self):
		return self.object.__hash__()

		def __getitem__(self, name):
		return self.object.__getitem__(name)

		def __iter__(self):
		return self.object.__iter__()

		def __getattr__(self, name):
		attr = getattr(self.object, name)
		if attr is None:
		raise AttributeError('%r object has no attribute %r' % (self.__class__, name))
		else:
		return attr

		def set(self, name, value):
		""" Set a response value for the specified tag.
		"""
		self._tags[name] = value

		def get(self, name):
		""" Get a response value for the specified tag.
		"""
		return self._tags[name]

		def copy(self):
		return copy(self)

		@property
		def show(self):
		return self._show

		@show.setter
		def show(self, show):
		self._show = show

		@property
		def show_tags(self):
		for tag in self.show:
		tag = self.object.get(tag)
		if tag is not None:
		yield tag

		def to_dict(self, tags=None):
		if tags is None:
		tags = self._show
		return dict((k, unicode(v)) for k, v in self._tags.iteritems() if k in tags)


		class TqlResponse(object):

		""" A set of TqlResponseObject returned after a query.
		"""

		def __init__(self, requestor, copy_from=None):
		self._requestor = requestor
		if copy_from is None:
		self._objects = OrderedDict()
		else:
		self._objects = OrderedDict((obj.id, obj) for obj in copy_from)

		def __repr__(self):
		return '<TqlResponse [%s]>' % ', '.join(self.objects)

		def __iter__(self):
		return self._objects.itervalues()

		def __contains__(self, obj):
		return self.has(obj)

		def __and__(self, other):
		response = TqlResponse(self._requestor)
		for obj in other:
		if obj in self:
		obj = obj.copy()
		obj.show \|= self[obj.id].show
		response.add(obj)
		return response

		def __or__(self, other):
		response = TqlResponse(self._requestor)
		for obj in other:
		if obj in self:
		obj = obj.copy()
		obj.show \|= self[obj.id].show
		response.add(obj)
		for obj in self:
		if obj not in response:
		response.add(obj)
		return response

		def __getitem__(self, name):
		obj = self.get(name)
		if obj is None:
		raise KeyError('Unknown object %s' % name)
		return obj

		@property
		def objects(self):
		""" Get the dict of object stored in this response.
		"""
		return self._objects

		def set_requestor(self, requestor):
		""" Set a new requestor on this TqlResponse object.
		"""
		self._requestor = requestor

		def get(self, object_id, default=None):
		""" Return the TqlResponseObject with the specified id, or default.
		"""
		if isinstance(object_id, TqlResponseObject):
		object_id = object_id.id
		return self._objects.get(object_id, default)

		def has(self, object_id):
		""" Return True if the provided id (or TqlResponseObject) is found in
		this TqlResponse object.
		"""
		return self.get(object_id) is not None

		def copy(self):
		""" Copy this TqlResponse object into another one.
		"""
		return TqlResponse(self._requestor, self)

		def limit(self, start, end, step=None):
		""" Return a new TqlResponse with objects inside limit.
		"""
		return TqlResponse(list(self)[start:end:step])

		def sorted(self, tag, reverse=False):
		""" Return a new TqlResponse with objects sorted by provided tag.
		"""
		sorter = lambda obj: None if obj.object.get(tag) is None else obj.object.get(tag).value
		objects = tuple(sorted(self._objects.itervalues(), key=sorter, reverse=reverse))
		for obj in objects:
		obj.show.add(tag)
		return TqlResponse(self._requestor, objects)

		def filter(self, tag, cmp_func):
		""" Filter value of specified tag using the provided comparison function.
		"""
		self.fetch(tag)
		matching = TqlResponse(self._requestor)
		for obj in self._objects.itervalues():
		try:
		tag_value = obj.get(tag)
		except KeyError:
		continue
		else:
		if cmp_func(tag_value):
		matching.add(obj)
		obj.show.add(tag)
		return matching

		def fetch(self, tags):
		""" Fetch some tags.
		"""
		if isinstance(tags, basestring):
		tags = (tags,)
		self._requestor.fetch(self._objects.itervalues(), tags)

		def add(self, obj):
		""" Add an object in the response.

		:param obj: the object to add to the response
		"""
		if obj not in self._objects:
		self._objects[obj.id] = obj

		#
		# Database:
		#

		class TqlDatabase(object):

		def __init__(self, default_requestor=None):
		self._objects = {}
		if default_requestor is None:
		self._default_requestor = StaticRequestor()
		else:
		self._default_requestor = default_requestor

		@property
		def objects(self):
		return self._objects.itervalues()

		def register(self, obj):
		""" Register a new object in the database.
		"""
		if obj.id not in self._objects:
		self._objects[obj.id] = obj

		def unregister(self, obj):
		""" Unregister an object from the database.
		"""
		if isinstance(obj, TqlObject):
		obj = obj.id
		del self._objects[obj]

		def get(self, object_id):
		return self._objects[object_id]

		def raw_query(self, tql, requestor=None):
		if requestor is None:
		requestor = self._default_requestor
		if isinstance(tql, basestring):
		tql = TqlParser(tql, debug=False, write_tables=False,
		errorlog=yacc.NullLogger()).parse()
		return self._evaluate_ast(tql, requestor)

		def query(self, tql, show=None, requestor=None):
		if requestor is None:
		requestor = self._default_requestor
		result = self.raw_query(tql, requestor)
		objects = []
		if show is None:
		show = ()
		for obj in result:
		tags_to_show = set(obj.itermatchingtags(show)) \| set(obj.show_tags)
		requestor.fetch((obj,), [t.name for t in tags_to_show])
		objects.append(obj.to_dict([t.name for t in tags_to_show]))
		return objects

		def get_by_id(self, object_id, tags=None, requestor=None):
		""" Get an object result including specified tags (or all if
		not specified).
		"""
		if requestor is None:
		requestor = self._default_requestor
		if tags is None:
		tags = ('*',)

		obj = self._objects.get(object_id)
		if obj is None:
		return None
		tags = [t.name for t in obj.itermatchingtags(tags)]
		robj = TqlResponseObject(obj)
		requestor.fetch((robj,), tags)
		return robj.to_dict(tags)

		#
		# TQL AST evaluation:
		#

		def _evaluate_ast(self, ast, requestor):
		objects = TqlResponse(requestor, (TqlResponseObject(obj) for obj in self._objects.itervalues()))
		return self._evaluate_expression(objects, ast.expression)

		def _evaluate_expression(self, objects, expression):
		if isinstance(expression, Filter):
		return self._evaluate_filter(objects, expression)
		elif isinstance(expression, FilterPresence):
		return self._evaluate_filter_presence(objects, expression)
		elif isinstance(expression, UnionOperator):
		return self._evaluate_union(objects, expression)
		elif isinstance(expression, IntersectionOperator):
		return self._evaluate_intersection(objects, expression)
		elif isinstance(expression, ShowOperator):
		return self._evaluate_show(objects, expression)
		elif isinstance(expression, SortOperator):
		return self._evaluate_sort(objects, expression)
		elif isinstance(expression, LimitOperator):
		return self._evaluate_limit(objects, expression)

		def _evaluate_filter(self, objects, filter):
		return objects.filter(filter.name, filter.match)

		def _evaluate_filter_presence(self, objects, filter):
		matching = TqlResponse(objects._requestor)
		for obj in objects:
		if (filter.name in obj and not filter.invert) or (filter.name not in obj and filter.invert):
		matching.add(obj)
		obj.show.add(filter.name)
		return matching

		def _evaluate_union(self, objects, union):
		left = self._evaluate_expression(objects, union.left_expression)
		right = self._evaluate_expression(objects, union.right_expression)
		return left \| right

		def _evaluate_intersection(self, objects, intersection):
		left = self._evaluate_expression(objects, intersection.left_expression)
		right = self._evaluate_expression(objects, intersection.right_expression)
		return left & right

		def _evaluate_show(self, objects, show):
		objects = self._evaluate_expression(objects, show.expression)
		for obj in objects:
		if show.invert:
		for tag in obj.object:
		if fnmatch(tag.name, show.pattern):
		obj.show.discard(tag.name)
		else:
		for tag in obj.object:
		if fnmatch(tag.name, show.pattern):
		obj.show.add(tag.name)
		return objects

		def _evaluate_sort(self, objects, sorter):
		objects = self._evaluate_expression(objects, sorter.expression)
		return objects.sorted(sorter.name, sorter.invert)

		def _evaluate_limit(self, objects, limit):
		objects = self._evaluate_expression(objects, limit.expression)
		return objects.limit(limit.start, limit.stop)

cloudcontrol/common/tql/db/object.py

0 → 100644

+81 −0

Original line number	Diff line number	Diff line
		""" Objects as stored in the Tql database.
		"""

		from fnmatch import fnmatch

		from cloudcontrol.common.tql.db.tag import StaticTag


		class TqlObject(object):

		""" An object in the TQL database.
		"""

		def __init__(self, id_):
		id_ = StaticTag('id', id_)
		self._tags = {'id': id_}

		def __repr__(self):
		return '<TqlObject id=%s>' % self.id

		def __contains__(self, value):
		return value in self._tags

		def __hash__(self):
		return hash(self._tags['id'].value)

		def __getitem__(self, name):
		return self._tags[name]

		def __iter__(self):
		return iter(self.itertags())

		@property
		def id(self):
		""" Get the id of the object.
		"""
		return self._tags['id'].value

		def itertags(self):
		""" Iter over tags registered on this object.
		"""
		for tag in self._tags.itervalues():
		yield tag

		def itermatchingtags(self, patterns):
		""" Iter over tags matching given patterns.
		"""
		if isinstance(patterns, basestring):
		patterns = (patterns,)

		for tag in self._tags.itervalues():
		for pattern in patterns:
		if fnmatch(tag.name, pattern):
		yield tag
		break

		def get(self, name, default=None):
		""" Get the tag with the specified name (or default).
		"""
		return self._tags.get(name, default)

		def get_value(self, name, default=None):
		""" Get the value of the tag with the specified name (or default).
		"""
		tag = self.get(name, None)
		return tag if tag is not None else default

		def register(self, tag):
		""" Register a tag on this object.
		"""

		if tag.name in self._tags:
		raise KeyError('A tag with this name is '
		'already registered on this object')
		else:
		self._tags[tag.name] = tag

		def unregister(self, name):
		""" Unregister the tag with specified name.
		"""
		del self._tags[name]

cloudcontrol/common/tql/db/requestor.py

0 → 100644

+81 −0

Original line number	Diff line number	Diff line
		""" Requestors are components in charge of updating list of tags on a list of
		objects.
		"""

		from collections import defaultdict

		from cloudcontrol.common.tql.db.tag import StaticTagInterface


		def fetcher(*args):

		""" Decorator used to create a fetching method.
		"""

		def decorator(func):
		func._fetcher_interfaces = args
		return func

		return decorator


		class BaseRequestor(object):

		""" Base class for all requestors.
		"""

		def __init__(self):
		self._fetcher_interface_mapping = {}
		for attr_name in dir(self):
		attr = getattr(self, attr_name)
		interfaces = getattr(attr, '_fetcher_interfaces', None)
		if interfaces is not None:
		for interface in interfaces:
		if interface in self._fetcher_interface_mapping:
		continue
		self._fetcher_interface_mapping[interface] = attr

		def fetch(self, objects, tags):
		""" This method allow to fetch tags on objects.
		"""
		map = defaultdict(lambda: [])
		for obj in objects:
		for tag_name in tags:
		try:
		tag = obj[tag_name]
		except KeyError:
		continue
		map[obj].append(tag)
		return self.fetch_map(map)

		def fetch_map(self, map):
		""" Fetch tags for specified objects.
		"""
		# Dispatch each tag to a fetcher according to its type:
		fetcher_map = defaultdict(lambda: defaultdict(lambda: []))
		for obj, tags in map.iteritems():
		for tag in tags:
		# Search the matching fetcher:
		for interface, fetcher in self._fetcher_interface_mapping.iteritems():
		if isinstance(tag, interface):
		fetcher_map[fetcher][obj].append(tag)
		else:
		pass #XXX: fetcher not found
		# Call each fetcher:
		for fetcher, map in fetcher_map.iteritems():
		fetcher(map)


		class StaticRequestor(BaseRequestor):

		""" A requestion which is able to request value for tags implementing the
		static interface.
		"""

		@fetcher(StaticTagInterface)
		def fetch_static(self, map):
		""" Fetch tag using the StaticTagInterface.
		"""
		for obj, tags in map.iteritems():
		for tag in tags:
		obj.set(tag.name, tag.value)

cloudcontrol/common/tql/db/tag.py

0 → 100644

+141 −0

Original line number	Diff line number	Diff line
		""" In-database tags representation.
		"""

		from abc import ABCMeta, abstractproperty
		from datetime import datetime, timedelta


		class BaseTagInterface(object):

		""" Base class for all tag interfaces.
		"""

		__metaclass__ = ABCMeta


		class BaseTag(object):

		""" Base class for a tag in the database.

		:param name: name of the tag
		"""

		interface = None # Define the tag fetching interface of the tag

		def __init__(self, name, **kwargs):
		self._name = name

		def __repr__(self):
		return '<%s %s>' % (self.__class__.__name__, self.name)

		def __hash__(self):
		return hash(self.name)

		#
		# Publics:
		#

		@property
		def name(self):
		return self._name


		#
		# Static tags
		#

		class StaticTagInterface(BaseTagInterface):

		""" An interface used to get static value stored by tags.
		"""

		@abstractproperty
		def value(self):
		""" The value of the tag.
		"""

		@value.setter
		def value(self, value):
		""" Set the value of the tag.
		"""


		class StaticTag(BaseTag):

		""" Static tag in the database.
		"""

		interface = 'static'

		def __init__(self, name, value='', **kwargs):
		super(StaticTag, self).__init__(name, **kwargs)
		self._value = value

		#
		# Implentation of StaticTagInterface:
		#

		@property
		def value(self):
		return self._value

		@value.setter
		def value(self, value):
		self._value = value


		StaticTagInterface.register(StaticTag)

		class CallbackTag(BaseTag, StaticTagInterface):

		""" A tag which is defined after a call to a callback.
		"""

		interface = 'static'

		def __init__(self, name, callback, ttl=None, extra={}, **kwargs):
		super(CallbackTag, self).__init__(name, **kwargs)
		self._callback = callback
		self._callback_extra = extra
		self._ttl = ttl
		self._cache_value = ''
		self._cache_last_update = None

		#
		# Implentation of StaticTagInterface:
		#

		@property
		def value(self):
		if self.ttl is not None or self.ttl is None and self._cache_last_update is None:
		if self._cache_last_update is None or datetime.now() - self._cache_last_update > self.ttl:
		self.value = str(self.callback(**self._callback_extra))
		return self._cache_value

		@value.setter
		def value(self, value):
		self._cache_last_update = datetime.now()
		self._cache_value = value

		#
		# Publics:
		#

		@property
		def callback(self):
		return self._callback

		@callback.setter
		def callback(self, callback):
		self._callback = callback

		@property
		def ttl(self):
		return timedelta(seconds=self._ttl) if self._ttl is not None else None

		@ttl.setter
		def ttl(self, ttl):
		self._ttl = ttl

		def invalidate(self):
		self._cache_last_update = None