Change config parser to configobj (8480fee9) · Commits · Seblu / installsystems

debian/control

+1 −1

Original line number	Diff line number	Diff line
		@@ -15,7 +15,7 @@ Description: Python2 Installation framework

		Package: python-installsystems
		Architecture: all
		Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2
		Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj
		XB-Python-Version: ${python:Versions}
		Description: Python2 Installation framework - Python2 modules
		This package provides InstallSystems Python modules

installsystems/config.py

+95 −103

Original line number	Diff line number	Diff line
		@@ -24,11 +24,46 @@ import codecs
		import os
		import sys
		from argparse import Namespace
		from ConfigParser import RawConfigParser
		from configobj import ConfigObj, flatten_errors
		from validate import Validator
		from installsystems.exception import *
		from installsystems.printer import *
		from installsystems.repository import RepositoryConfig

		# This must not be an unicode string, because configobj don't decode configspec
		# with the provided encoding
		MAIN_CONFIG_SPEC = """\
		[installsystems]
		verbosity = integer(0, 2)
		repo_config = string
		repo_search = string
		repo_filter = string
		repo_timeout = integer
		cache = string(default=%s)
		timeout = integer
		no_cache = boolean
		no_check = boolean
		no-sync = boolean
		no_color = boolean
		nice = integer
		ionice_class = option("none", "rt", "be", "idle")
		ionice_level = integer
		"""

		# This must not be an unicode string, because configobj don't decode configspec
		# with the provided encoding
		REPO_CONFIG_SPEC = """\
		[__many__]
		path = string
		fmod = integer
		dmod = integer
		uid = string
		gid = string
		offline = boolean
		lastpath = string
		dbpath = string
		"""


		class ConfigFile(object):
		'''
		@@ -37,20 +72,37 @@ class ConfigFile(object):

		def __init__(self, filename):
		'''
		filename can be full path to config file or a name in config directory
		Filename can be full path to config file or a name in config directory
		'''
		# try to get filename in default config dir
		if os.path.isfile(filename):
		self.path = os.path.abspath(filename)
		else:
		self.path = self._config_path(filename)
		self.reload()

		def reload():
		'''
		Reload configuration from file
		'''
		raise NotImplementedError
		# loading config file if exists
		if self.path is None:
		raise ISWarning("No config file to load")
		self.config = ConfigObj(self.path, configspec=self.configspec,
		encoding="utf8", file_error=True)
		self.validate()

		def validate(self):
		'''
		Validate the configuration file according to the configuration specification
		If some values doesn't respect specification, she's ignored and a warning is issued.
		'''
		res = self.config.validate(Validator(), preserve_errors=True)
		# If everything is fine, the validation return True
		# Else, it returns a list of (section, optname, error)
		if res is not True:
		for section, optname, error in flatten_errors(self.config, res):
		# If error is False, this mean no value as been supplied,
		# so we use the default value
		# Else, the check has failed
		if error:
		warn("%s: %s Skipped" % (optname, error))
		# remove wrong value to avoid merging it with argparse value
		del self.config[section[0]][optname]

		def _config_path(self, name):
		'''
		@@ -65,91 +117,20 @@ class ConfigFile(object):

		class MainConfigFile(ConfigFile):
		'''
		Program configuration file
		'''

		valid_options = {
		"verbosity": [0,1,2],
		"no_cache": bool,
		"no_color": bool,
		"timeout": int,
		"cache": str,
		"repo_search": str,
		"repo_filter": str,
		"repo_config": str,
		"repo_timeout": int,
		"nice": int,
		"ionice_class": ["none", "rt", "be", "idle"],
		"ionice_level": int
		}
		Program configuration class
		'''

		def __init__(self, filename, prefix=os.path.basename(sys.argv[0])):
		self.prefix = prefix
		ConfigFile.__init__(self, filename)

		def reload(self):
		'''
		Load/Reload config file
		'''
		self._config = {}
		# loading default options
		self._config["cache"] = self.cache
		# loading config file if exists
		if self.path is None:
		debug("No main config file to load")
		return
		debug(u"Loading main config file: %s" % self.path)
		self.configspec = (MAIN_CONFIG_SPEC % self.cache).splitlines()
		try:
		cp = RawConfigParser()
		cp.read(self.path)
		# main configuration
		if cp.has_section(self.prefix):
		self._config.update(cp.items(self.prefix))
		super(MainConfigFile, self).__init__(filename)
		debug(u"Loading main config file: %s" % self.path)
		except ISWarning:
		debug("No main config file to load")
		except Exception as e:
		raise ISError(u"Unable load main config file %s" % self.path, e)

		def parse(self, namespace=None):
		'''
		Parse current loaded option within a namespace
		'''
		if namespace is None:
		namespace = Namespace()
		for option, value in self._config.items():
		# check option is valid
		if option not in self.valid_options.keys():
		warn(u"Invalid option %s in %s, skipped" % (option, self.path))
		continue
		# we expect a string like
		if not isinstance(option, basestring):
		raise TypeError(u"Invalid config parser option %s type" % option)
		# smartly cast option's value
		if self.valid_options[option] is bool:
		value = value.strip().lower() not in ("false", "no", "0", "")
		# in case of valid option is a list, we take the type of the first
		# argument of the list to convert value into it
		# as a consequence, all element of a list must be of the same type!
		# empty list are forbidden !
		elif isinstance(self.valid_options[option], list):
		ctype = type(self.valid_options[option][0])
		try:
		value = ctype(value)
		except ValueError:
		warn("Invalid option %s type (must be %s), skipped" %
		(option, ctype))
		continue
		if value not in self.valid_options[option]:
		warn("Invalid value %s in option %s (must be in %s), skipped" %
		(value, option, self.valid_options[option]))
		continue
		else:
		try:
		value = self.valid_options[option](value)
		except ValueError:
		warn("Invalid option %s type (must be %s), skipped" %
		(option, self.valid_options[option]))
		continue
		setattr(namespace, option, value)
		return namespace

		def _cache_paths(self):
		'''
		List all candidates to cache directories. Alive or not
		@@ -177,8 +158,6 @@ class MainConfigFile(ConfigFile):
		'''
		Find a cache directory
		'''
		if "cache" in self._config:
		return self._config["cache"]
		if self._cache_path() is None:
		for di in self._cache_paths():
		try:
		@@ -188,34 +167,47 @@ class MainConfigFile(ConfigFile):
		debug(u"Unable to create %s: %s" % (di, e))
		return self._cache_path()

		def parse(self, namespace=None):
		'''
		Parse current loaded option within a namespace
		'''
		if namespace is None:
		namespace = Namespace()
		if self.path:
		for option, value in self.config[self.prefix].items():
		setattr(namespace, option, value)
		return namespace


		class RepoConfigFile(ConfigFile):
		'''
		Repository Configuration class
		'''

		def reload(self):
		'''
		Load/Reload config file
		'''
		def __init__(self, filename):
		# seting default config
		self._config = {}
		self._repos = []
		# if no file nothing to load
		if self.path is None:
		return
		# loading config file if exists
		self.configspec = REPO_CONFIG_SPEC.splitlines()
		try:
		super(RepoConfigFile, self).__init__(filename)
		debug(u"Loading repository config file: %s" % self.path)
		self._parse()
		except ISWarning:
		debug("No repository config file to load")

		def _parse(self):
		'''
		Parse repositories from config
		'''
		try:
		cp = RawConfigParser()
		cp.readfp(codecs.open(self.path, "r", "utf8"))
		# each section is a repository
		for rep in cp.sections():
		for rep in self.config.sections:
		# check if its a repo section
		if "path" not in cp.options(rep):
		if "path" not in self.config[rep]:
		continue
		# get all options in repo
		self._repos.append(RepositoryConfig(rep, **dict(cp.items(rep))))
		self._repos.append(RepositoryConfig(rep, **dict(self.config[rep].items())))
		except Exception as e:
		raise ISError(u"Unable to load repository file %s" % self.path, e)

installsystems/image.py

+50 −19

Original line number	Diff line number	Diff line
		@@ -22,7 +22,7 @@ Image stuff
		'''

		import codecs
		import ConfigParser
		import configobj
		import cStringIO
		import difflib
		import imp
		@@ -37,15 +37,28 @@ import subprocess
		import sys
		import tarfile
		import time
		import validate
		import installsystems
		import installsystems.template as istemplate
		import installsystems.tools as istools
		from installsystems.ordereddict import OrderedDict
		from installsystems.exception import *
		from installsystems.printer import *
		from installsystems.tools import PipeFile
		from installsystems.tarball import Tarball


		# This must not be an unicode string, because configobj don't decode configspec
		# with the provided encoding
		DESCRIPTION_CONFIG_SPEC = """\
		[image]
		name = IS_name
		version = IS_version
		description = string
		author = string
		is_min_version = IS_min_version
		"""


		class Image(object):
		'''
		Abstract class of images
		@@ -63,6 +76,9 @@ class Image(object):
		'''
		if re.match("^[-_\w]+$", buf) is None:
		raise ISError(u"Invalid image name %s" % buf)
		# return the image name, because this function is used by ConfigObj
		# validate to ensure the image name is correct
		return buf

		@staticmethod
		def check_image_version(buf):
		@@ -71,6 +87,21 @@ class Image(object):
		'''
		if re.match("^\d+(\.\d+)(([~+]).)?$", buf) is None:
		raise ISError(u"Invalid image version %s" % buf)
		# return the image version, because this function is used by ConfigObj
		# validate to ensure the image version is correct
		return buf

		@staticmethod
		def check_min_version(version):
		'''
		Check InstallSystems min version
		'''
		if istools.compare_versions(installsystems.version, version) < 0:
		raise ISError("Minimum Installsystems version not satisfied "
		"(%s)" % version)
		# return the version, because this function is used by ConfigObj
		# validate to ensure the version is correct
		return version

		@staticmethod
		def compare_versions(v1, v2):
		@@ -644,23 +675,23 @@ class SourceImage(Image):
		d = dict()
		try:
		descpath = os.path.join(self.base_path, "description")
		cp = ConfigParser.RawConfigParser(dict_type=OrderedDict)
		cp.readfp(codecs.open(descpath, "r", "UTF-8"))
		for n in ("name","version", "description", "author"):
		d[n] = cp.get("image", n)
		# get min image version
		if cp.has_option("image", "is_min_version"):
		d["is_min_version"] = cp.get("image", "is_min_version")
		else:
		d["is_min_version"] = 0
		# check image name
		self.check_image_name(d["name"])
		# check image version
		self.check_image_version(d["version"])
		# check installsystems min version
		if self.compare_versions(installsystems.version, d["is_min_version"]) < 0:
		raise ISError("Minimum Installsystems version not satisfied "
		"(%s)" % d["is_min_version"])
		cp = configobj.ConfigObj(descpath,
		configspec=DESCRIPTION_CONFIG_SPEC.splitlines(),
		encoding="utf8", file_error=True)
		res = cp.validate(validate.Validator({"IS_name": Image.check_image_name,
		"IS_version": Image.check_image_version,
		"IS_min_version": Image.check_min_version}), preserve_errors=True)
		# If everything is fine, the validation return True
		# Else, it returns a list of (section, optname, error)
		if res is not True:
		for section, optname, error in configobj.flatten_errors(cp, res):
		# If error is False, this mean no value as been supplied,
		# so we use the default value
		# Else, the check has failed
		if error:
		installsystems.printer.error('Wrong description file, %s %s: %s' % (section, optname, error))
		for n in ("name","version", "description", "author", "is_min_version"):
		d[n] = cp["image"][n]
		except Exception as e:
		raise ISError(u"Bad description", e)
		return d

installsystems/ordereddict.py

deleted100644 → 0

+0 −284

Original line number	Diff line number	Diff line
		# -- python --
		# -- coding: utf-8 --

		# Copyright (c) 2009 Raymond Hettinger

		# Permission is hereby granted, free of charge, to any person obtaining a copy
		# of this software and associated documentation files (the "Software"), to deal
		# in the Software without restriction, including without limitation the rights to
		# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
		# of the Software, and to permit persons to whom the Software is furnished to do
		# so, subject to the following conditions:

		# The above copyright notice and this permission notice shall be included in
		# all copies or substantial portions of the Software.

		# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
		# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
		# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
		# SOFTWARE.


		'''
		Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
		Passes Python2.7's test suite and incorporates all the latest updates.
		'''

		try:
		from thread import get_ident as _get_ident
		except ImportError:
		from dummy_thread import get_ident as _get_ident

		try:
		from _abcoll import KeysView, ValuesView, ItemsView
		except ImportError:
		pass


		class OrderedDict(dict):
		'Dictionary that remembers insertion order'
		# An inherited dict maps keys to values.
		# The inherited dict provides __getitem__, __len__, __contains__, and get.
		# The remaining methods are order-aware.
		# Big-O running times for all methods are the same as for regular dictionaries.

		# The internal self.__map dictionary maps keys to links in a doubly linked list.
		# The circular doubly linked list starts and ends with a sentinel element.
		# The sentinel element never gets deleted (this simplifies the algorithm).
		# Each link is stored as a list of length three: [PREV, NEXT, KEY].

		def __init__(self, args, *kwds):
		'''Initialize an ordered dictionary. Signature is the same as for
		regular dictionaries, but keyword arguments are not recommended
		because their insertion order is arbitrary.

		'''
		if len(args) > 1:
		raise TypeError('expected at most 1 arguments, got %d' % len(args))
		try:
		self.__root
		except AttributeError:
		self.__root = root = [] # sentinel node
		root[:] = [root, root, None]
		self.__map = {}
		self.__update(args, *kwds)

		def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
		'od.__setitem__(i, y) <==> od[i]=y'
		# Setting a new item creates a new link which goes at the end of the linked
		# list, and the inherited dictionary is updated with the new key/value pair.
		if key not in self:
		root = self.__root
		last = root[0]
		last[1] = root[0] = self.__map[key] = [last, root, key]
		dict_setitem(self, key, value)

		def __delitem__(self, key, dict_delitem=dict.__delitem__):
		'od.__delitem__(y) <==> del od[y]'
		# Deleting an existing item uses self.__map to find the link which is
		# then removed by updating the links in the predecessor and successor nodes.
		dict_delitem(self, key)
		link_prev, link_next, key = self.__map.pop(key)
		link_prev[1] = link_next
		link_next[0] = link_prev

		def __iter__(self):
		'od.__iter__() <==> iter(od)'
		root = self.__root
		curr = root[1]
		while curr is not root:
		yield curr[2]
		curr = curr[1]

		def __reversed__(self):
		'od.__reversed__() <==> reversed(od)'
		root = self.__root
		curr = root[0]
		while curr is not root:
		yield curr[2]
		curr = curr[0]

		def clear(self):
		'od.clear() -> None. Remove all items from od.'
		try:
		for node in self.__map.itervalues():
		del node[:]
		root = self.__root
		root[:] = [root, root, None]
		self.__map.clear()
		except AttributeError:
		pass
		dict.clear(self)

		def popitem(self, last=True):
		'''od.popitem() -> (k, v), return and remove a (key, value) pair.
		Pairs are returned in LIFO order if last is true or FIFO order if false.

		'''
		if not self:
		raise KeyError('dictionary is empty')
		root = self.__root
		if last:
		link = root[0]
		link_prev = link[0]
		link_prev[1] = root
		root[0] = link_prev
		else:
		link = root[1]
		link_next = link[1]
		root[1] = link_next
		link_next[0] = root
		key = link[2]
		del self.__map[key]
		value = dict.pop(self, key)
		return key, value

		# -- the following methods do not depend on the internal structure --

		def keys(self):
		'od.keys() -> list of keys in od'
		return list(self)

		def values(self):
		'od.values() -> list of values in od'
		return [self[key] for key in self]

		def items(self):
		'od.items() -> list of (key, value) pairs in od'
		return [(key, self[key]) for key in self]

		def iterkeys(self):
		'od.iterkeys() -> an iterator over the keys in od'
		return iter(self)

		def itervalues(self):
		'od.itervalues -> an iterator over the values in od'
		for k in self:
		yield self[k]

		def iteritems(self):
		'od.iteritems -> an iterator over the (key, value) items in od'
		for k in self:
		yield (k, self[k])

		def update(args, *kwds):
		'''od.update(E, **F) -> None. Update od from dict/iterable E and F.

		If E is a dict instance, does: for k in E: od[k] = E[k]
		If E has a .keys() method, does: for k in E.keys(): od[k] = E[k]
		Or if E is an iterable of items, does: for k, v in E: od[k] = v
		In either case, this is followed by: for k, v in F.items(): od[k] = v

		'''
		if len(args) > 2:
		raise TypeError('update() takes at most 2 positional '
		'arguments (%d given)' % (len(args),))
		elif not args:
		raise TypeError('update() takes at least 1 argument (0 given)')
		self = args[0]
		# Make progressively weaker assumptions about "other"
		other = ()
		if len(args) == 2:
		other = args[1]
		if isinstance(other, dict):
		for key in other:
		self[key] = other[key]
		elif hasattr(other, 'keys'):
		for key in other.keys():
		self[key] = other[key]
		else:
		for key, value in other:
		self[key] = value
		for key, value in kwds.items():
		self[key] = value

		__update = update # let subclasses override update without breaking __init__

		__marker = object()

		def pop(self, key, default=__marker):
		'''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
		If key is not found, d is returned if given, otherwise KeyError is raised.

		'''
		if key in self:
		result = self[key]
		del self[key]
		return result
		if default is self.__marker:
		raise KeyError(key)
		return default

		def setdefault(self, key, default=None):
		'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
		if key in self:
		return self[key]
		self[key] = default
		return default

		def __repr__(self, _repr_running={}):
		'od.__repr__() <==> repr(od)'
		call_key = id(self), _get_ident()
		if call_key in _repr_running:
		return '...'
		_repr_running[call_key] = 1
		try:
		if not self:
		return '%s()' % (self.__class__.__name__,)
		return '%s(%r)' % (self.__class__.__name__, self.items())
		finally:
		del _repr_running[call_key]

		def __reduce__(self):
		'Return state information for pickling'
		items = [[k, self[k]] for k in self]
		inst_dict = vars(self).copy()
		for k in vars(OrderedDict()):
		inst_dict.pop(k, None)
		if inst_dict:
		return (self.__class__, (items,), inst_dict)
		return self.__class__, (items,)

		def copy(self):
		'od.copy() -> a shallow copy of od'
		return self.__class__(self)

		@classmethod
		def fromkeys(cls, iterable, value=None):
		'''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
		and values equal to v (which defaults to None).

		'''
		d = cls()
		for key in iterable:
		d[key] = value
		return d

		def __eq__(self, other):
		'''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
		while comparison to a regular mapping is order-insensitive.

		'''
		if isinstance(other, OrderedDict):
		return len(self)==len(other) and self.items() == other.items()
		return dict.__eq__(self, other)

		def __ne__(self, other):
		return not self == other

		# -- the following methods are only used in Python 2.7 --

		def viewkeys(self):
		"od.viewkeys() -> a set-like object providing a view on od's keys"
		return KeysView(self)

		def viewvalues(self):
		"od.viewvalues() -> an object providing a view on od's values"
		return ValuesView(self)

		def viewitems(self):
		"od.viewitems() -> a set-like object providing a view on od's items"
		return ItemsView(self)