From 8480fee99d6102e1bd90a7d282f85d615b543911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= Date: Mon, 16 Jul 2012 17:27:01 +0200 Subject: [PATCH] Change config parser to configobj MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit configobj keeps the order of options in config file, we don't need to use OrderedDict. Seblu: Sad :'( Signed-off-by: Sébastien Luttringer --- debian/control | 2 +- installsystems/config.py | 198 ++++++++++++------------ installsystems/image.py | 69 ++++++--- installsystems/ordereddict.py | 284 ---------------------------------- 4 files changed, 146 insertions(+), 407 deletions(-) delete mode 100644 installsystems/ordereddict.py diff --git a/debian/control b/debian/control index aa16403..2faeeb6 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Description: Python2 Installation framework Package: python-installsystems Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2 +Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj XB-Python-Version: ${python:Versions} Description: Python2 Installation framework - Python2 modules This package provides InstallSystems Python modules diff --git a/installsystems/config.py b/installsystems/config.py index e837178..a9cb958 100644 --- a/installsystems/config.py +++ b/installsystems/config.py @@ -24,11 +24,46 @@ import codecs import os import sys from argparse import Namespace -from ConfigParser import RawConfigParser +from configobj import ConfigObj, flatten_errors +from validate import Validator from installsystems.exception import * from installsystems.printer import * from installsystems.repository import RepositoryConfig +# This must not be an unicode string, because configobj don't decode configspec +# with the provided encoding +MAIN_CONFIG_SPEC = """\ +[installsystems] +verbosity = integer(0, 2) +repo_config = string +repo_search = string +repo_filter = string +repo_timeout = integer +cache = string(default=%s) +timeout = integer +no_cache = boolean +no_check = boolean +no-sync = boolean +no_color = boolean +nice = integer +ionice_class = option("none", "rt", "be", "idle") +ionice_level = integer +""" + +# This must not be an unicode string, because configobj don't decode configspec +# with the provided encoding +REPO_CONFIG_SPEC = """\ +[__many__] + path = string + fmod = integer + dmod = integer + uid = string + gid = string + offline = boolean + lastpath = string + dbpath = string +""" + class ConfigFile(object): ''' @@ -37,20 +72,37 @@ class ConfigFile(object): def __init__(self, filename): ''' - filename can be full path to config file or a name in config directory + Filename can be full path to config file or a name in config directory ''' - #try to get filename in default config dir + # try to get filename in default config dir if os.path.isfile(filename): self.path = os.path.abspath(filename) else: self.path = self._config_path(filename) - self.reload() - - def reload(): - ''' - Reload configuration from file - ''' - raise NotImplementedError + # loading config file if exists + if self.path is None: + raise ISWarning("No config file to load") + self.config = ConfigObj(self.path, configspec=self.configspec, + encoding="utf8", file_error=True) + self.validate() + + def validate(self): + ''' + Validate the configuration file according to the configuration specification + If some values doesn't respect specification, she's ignored and a warning is issued. + ''' + res = self.config.validate(Validator(), preserve_errors=True) + # If everything is fine, the validation return True + # Else, it returns a list of (section, optname, error) + if res is not True: + for section, optname, error in flatten_errors(self.config, res): + # If error is False, this mean no value as been supplied, + # so we use the default value + # Else, the check has failed + if error: + warn("%s: %s Skipped" % (optname, error)) + # remove wrong value to avoid merging it with argparse value + del self.config[section[0]][optname] def _config_path(self, name): ''' @@ -65,91 +117,20 @@ class ConfigFile(object): class MainConfigFile(ConfigFile): ''' - Program configuration file + Program configuration class ''' - valid_options = { - "verbosity": [0,1,2], - "no_cache": bool, - "no_color": bool, - "timeout": int, - "cache": str, - "repo_search": str, - "repo_filter": str, - "repo_config": str, - "repo_timeout": int, - "nice": int, - "ionice_class": ["none", "rt", "be", "idle"], - "ionice_level": int - } def __init__(self, filename, prefix=os.path.basename(sys.argv[0])): self.prefix = prefix - ConfigFile.__init__(self, filename) - - def reload(self): - ''' - Load/Reload config file - ''' - self._config = {} - # loading default options - self._config["cache"] = self.cache - # loading config file if exists - if self.path is None: - debug("No main config file to load") - return - debug(u"Loading main config file: %s" % self.path) + self.configspec = (MAIN_CONFIG_SPEC % self.cache).splitlines() try: - cp = RawConfigParser() - cp.read(self.path) - # main configuration - if cp.has_section(self.prefix): - self._config.update(cp.items(self.prefix)) + super(MainConfigFile, self).__init__(filename) + debug(u"Loading main config file: %s" % self.path) + except ISWarning: + debug("No main config file to load") except Exception as e: raise ISError(u"Unable load main config file %s" % self.path, e) - def parse(self, namespace=None): - ''' - Parse current loaded option within a namespace - ''' - if namespace is None: - namespace = Namespace() - for option, value in self._config.items(): - # check option is valid - if option not in self.valid_options.keys(): - warn(u"Invalid option %s in %s, skipped" % (option, self.path)) - continue - # we expect a string like - if not isinstance(option, basestring): - raise TypeError(u"Invalid config parser option %s type" % option) - # smartly cast option's value - if self.valid_options[option] is bool: - value = value.strip().lower() not in ("false", "no", "0", "") - # in case of valid option is a list, we take the type of the first - # argument of the list to convert value into it - # as a consequence, all element of a list must be of the same type! - # empty list are forbidden ! - elif isinstance(self.valid_options[option], list): - ctype = type(self.valid_options[option][0]) - try: - value = ctype(value) - except ValueError: - warn("Invalid option %s type (must be %s), skipped" % - (option, ctype)) - continue - if value not in self.valid_options[option]: - warn("Invalid value %s in option %s (must be in %s), skipped" % - (value, option, self.valid_options[option])) - continue - else: - try: - value = self.valid_options[option](value) - except ValueError: - warn("Invalid option %s type (must be %s), skipped" % - (option, self.valid_options[option])) - continue - setattr(namespace, option, value) - return namespace - def _cache_paths(self): ''' List all candidates to cache directories. Alive or not @@ -177,8 +158,6 @@ class MainConfigFile(ConfigFile): ''' Find a cache directory ''' - if "cache" in self._config: - return self._config["cache"] if self._cache_path() is None: for di in self._cache_paths(): try: @@ -188,34 +167,47 @@ class MainConfigFile(ConfigFile): debug(u"Unable to create %s: %s" % (di, e)) return self._cache_path() + def parse(self, namespace=None): + ''' + Parse current loaded option within a namespace + ''' + if namespace is None: + namespace = Namespace() + if self.path: + for option, value in self.config[self.prefix].items(): + setattr(namespace, option, value) + return namespace + class RepoConfigFile(ConfigFile): ''' Repository Configuration class ''' - def reload(self): - ''' - Load/Reload config file - ''' + def __init__(self, filename): # seting default config self._config = {} self._repos = [] - # if no file nothing to load - if self.path is None: - return - # loading config file if exists - debug(u"Loading repository config file: %s" % self.path) + self.configspec = REPO_CONFIG_SPEC.splitlines() + try: + super(RepoConfigFile, self).__init__(filename) + debug(u"Loading repository config file: %s" % self.path) + self._parse() + except ISWarning: + debug("No repository config file to load") + + def _parse(self): + ''' + Parse repositories from config + ''' try: - cp = RawConfigParser() - cp.readfp(codecs.open(self.path, "r", "utf8")) # each section is a repository - for rep in cp.sections(): + for rep in self.config.sections: # check if its a repo section - if "path" not in cp.options(rep): + if "path" not in self.config[rep]: continue # get all options in repo - self._repos.append(RepositoryConfig(rep, **dict(cp.items(rep)))) + self._repos.append(RepositoryConfig(rep, **dict(self.config[rep].items()))) except Exception as e: raise ISError(u"Unable to load repository file %s" % self.path, e) diff --git a/installsystems/image.py b/installsystems/image.py index ab91cf2..2a6d4c4 100644 --- a/installsystems/image.py +++ b/installsystems/image.py @@ -22,7 +22,7 @@ Image stuff ''' import codecs -import ConfigParser +import configobj import cStringIO import difflib import imp @@ -37,15 +37,28 @@ import subprocess import sys import tarfile import time +import validate import installsystems import installsystems.template as istemplate import installsystems.tools as istools -from installsystems.ordereddict import OrderedDict from installsystems.exception import * from installsystems.printer import * from installsystems.tools import PipeFile from installsystems.tarball import Tarball + +# This must not be an unicode string, because configobj don't decode configspec +# with the provided encoding +DESCRIPTION_CONFIG_SPEC = """\ +[image] +name = IS_name +version = IS_version +description = string +author = string +is_min_version = IS_min_version +""" + + class Image(object): ''' Abstract class of images @@ -63,6 +76,9 @@ class Image(object): ''' if re.match("^[-_\w]+$", buf) is None: raise ISError(u"Invalid image name %s" % buf) + # return the image name, because this function is used by ConfigObj + # validate to ensure the image name is correct + return buf @staticmethod def check_image_version(buf): @@ -71,6 +87,21 @@ class Image(object): ''' if re.match("^\d+(\.\d+)*(([~+]).*)?$", buf) is None: raise ISError(u"Invalid image version %s" % buf) + # return the image version, because this function is used by ConfigObj + # validate to ensure the image version is correct + return buf + + @staticmethod + def check_min_version(version): + ''' + Check InstallSystems min version + ''' + if istools.compare_versions(installsystems.version, version) < 0: + raise ISError("Minimum Installsystems version not satisfied " + "(%s)" % version) + # return the version, because this function is used by ConfigObj + # validate to ensure the version is correct + return version @staticmethod def compare_versions(v1, v2): @@ -644,23 +675,23 @@ class SourceImage(Image): d = dict() try: descpath = os.path.join(self.base_path, "description") - cp = ConfigParser.RawConfigParser(dict_type=OrderedDict) - cp.readfp(codecs.open(descpath, "r", "UTF-8")) - for n in ("name","version", "description", "author"): - d[n] = cp.get("image", n) - # get min image version - if cp.has_option("image", "is_min_version"): - d["is_min_version"] = cp.get("image", "is_min_version") - else: - d["is_min_version"] = 0 - # check image name - self.check_image_name(d["name"]) - # check image version - self.check_image_version(d["version"]) - # check installsystems min version - if self.compare_versions(installsystems.version, d["is_min_version"]) < 0: - raise ISError("Minimum Installsystems version not satisfied " - "(%s)" % d["is_min_version"]) + cp = configobj.ConfigObj(descpath, + configspec=DESCRIPTION_CONFIG_SPEC.splitlines(), + encoding="utf8", file_error=True) + res = cp.validate(validate.Validator({"IS_name": Image.check_image_name, + "IS_version": Image.check_image_version, + "IS_min_version": Image.check_min_version}), preserve_errors=True) + # If everything is fine, the validation return True + # Else, it returns a list of (section, optname, error) + if res is not True: + for section, optname, error in configobj.flatten_errors(cp, res): + # If error is False, this mean no value as been supplied, + # so we use the default value + # Else, the check has failed + if error: + installsystems.printer.error('Wrong description file, %s %s: %s' % (section, optname, error)) + for n in ("name","version", "description", "author", "is_min_version"): + d[n] = cp["image"][n] except Exception as e: raise ISError(u"Bad description", e) return d diff --git a/installsystems/ordereddict.py b/installsystems/ordereddict.py deleted file mode 100644 index 5e7ce16..0000000 --- a/installsystems/ordereddict.py +++ /dev/null @@ -1,284 +0,0 @@ -# -*- python -*- -# -*- coding: utf-8 -*- - -# Copyright (c) 2009 Raymond Hettinger - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -''' -Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -Passes Python2.7's test suite and incorporates all the latest updates. -''' - -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) -- GitLab