Commit 084ed9b5 authored by Seblu's avatar Seblu

Deep refactoring

- 2 new packages (image and repository).
- Split big modules into little one under packages.
- Replace full module import by needed objects
- Avoid wildcard imports
- Fix a lot of pylint error and warnings on modules
parent 118f3047
......@@ -28,20 +28,37 @@ dist_doc_DATA = README COPYRIGHT LICENSE AUTHORS DEPENDENCIES
dist_bin_SCRIPTS = bin/is
noinst_SCRIPTS = installsystems/__init__.py
# python library
# python installsystems package
installsystemsdir=$(pythondir)/installsystems
installsystems_PYTHON = \
installsystems/__init__.py \
installsystems/config.py \
installsystems/database.py \
installsystems/exception.py \
installsystems/image.py \
installsystems/printer.py \
installsystems/repository.py \
installsystems/tarball.py \
installsystems/template.py \
installsystems/tools.py
installsystemsimagedir=$(pythondir)/installsystems/image
installsystemsimage_PYTHON = \
installsystems/image/__init__.py \
installsystems/image/changelog.py \
installsystems/image/image.py \
installsystems/image/package.py \
installsystems/image/payload.py \
installsystems/image/source.py \
installsystems/image/tarball.py
installsystemsrepositorydir=$(pythondir)/installsystems/repository
installsystemsrepository_PYTHON = \
installsystems/repository/__init__.py \
installsystems/repository/config.py \
installsystems/repository/database.py \
installsystems/repository/factory.py \
installsystems/repository/manager.py \
installsystems/repository/repository.py \
installsystems/repository/repository1.py \
installsystems/repository/repository2.py
do_substitution = $(SED) -e 's,[@]pythondir[@],$(pythondir),g' \
-e 's,[@]PACKAGE[@],$(PACKAGE),g' \
-e 's,[@]VERSION[@],$(VERSION),g'
......
......@@ -20,25 +20,24 @@
InstallSystems Command line Tool
'''
import os
import datetime
import re
import fnmatch
import warnings
import argparse
import psutil
import socket
import sys
import installsystems
import installsystems.printer
import installsystems.tools as istools
from installsystems.exception import *
from installsystems.printer import *
from installsystems.repository import Repository
from installsystems.repository import RepositoryManager
from installsystems.repository import RepositoryConfig
from installsystems.image import PackageImage, SourceImage
from argparse import ArgumentParser
from datetime import timedelta
from installsystems import VERSION
from installsystems.config import MainConfigFile, RepoConfigFile
from installsystems.exception import ISError, ISException
from installsystems.image import PackageImage, SourceImage
from installsystems.printer import arrow, arrowlevel, setmode
from installsystems.printer import out, warn, error, debug, confirm
from installsystems.repository import Repository, RepositoryManager, RepositoryConfig
from installsystems.tools import chroot, prepare_chroot, unprepare_chroot
from installsystems.tools import isfile, smd5sum, argv
from os import getpid, getcwdu, chdir
from psutil import IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE
from psutil import Process, IOPRIO_CLASS_NONE
from socket import setdefaulttimeout
# used by os.path.isfile
import os
################################################################################
# Common functions
......@@ -52,15 +51,15 @@ def load_repositories(args):
if args.no_cache:
args.cache = None
# split filter and search in list
args.repo_filter = Repository.split_repository_list(args.repo_filter)
args.repo_search = Repository.split_repository_list(args.repo_search)
args.repo_filter = Repository.split_list(args.repo_filter)
args.repo_search = Repository.split_list(args.repo_search)
# init repo cache object
repoman = RepositoryManager(args.cache, timeout=args.repo_timeout or args.timeout,
filter=args.repo_filter, search=args.repo_search)
# register repositories (order matter)
# load repo configs from command line
if args.repo_path != "":
repoconf = RepositoryConfig(istools.smd5sum(args.repo_path)[:8],
repoconf = RepositoryConfig(smd5sum(args.repo_path)[:8],
path=args.repo_path)
repoman.register(repoconf, temp=True, nosync=args.no_sync)
# load repo configs from config
......@@ -80,7 +79,7 @@ def get_images(patterns, repoman, local=True, min=None, max=None):
ans = []
for pattern in patterns:
# check if image is a local file
if local and istools.isfile(pattern) and os.path.isfile(pattern):
if local and isfile(pattern) and os.path.isfile(pattern):
ans.append((pattern, None))
else: # we need to find image in a repository
ans += sorted(repoman.select_images([pattern]).items())
......@@ -128,8 +127,8 @@ def c_build(args):
arrow("Build %s" % path)
# chdir inside path if --chdir
if args.chdir:
cwd = os.getcwdu()
os.chdir(path)
cwd = getcwdu()
chdir(path)
path = "."
arrowlevel(1)
# load source image
......@@ -138,19 +137,19 @@ def c_build(args):
dt = simg.build(force=args.force, force_payload=args.payload,
check=not args.no_check, script=not args.no_script)
gdt += dt
arrow(u"Build time: %s" % datetime.timedelta(seconds=dt))
arrow(u"Build time: %s" % timedelta(seconds=dt))
if args.chdir:
os.chdir(cwd)
chdir(cwd)
arrowlevel(-1)
if len(args.paths) > 1:
arrow(u"Global build time: %s" % datetime.timedelta(seconds=gdt))
arrow(u"Global build time: %s" % timedelta(seconds=gdt))
def c_cat(args):
'''
Display files inside a packaged image
'''
repoman = load_repositories(args)
image, repo = next(get_images([args.pattern], repoman, min=1, max=1))
image = next(get_images([args.pattern], repoman, min=1, max=1))[0]
for filename in args.file:
image.cat(filename)
......@@ -160,7 +159,7 @@ def c_changelog(args):
'''
repoman = load_repositories(args)
images = list(get_images(args.pattern, repoman, min=1))
for image, repo in images:
for image in images:
if len(images) > 1:
out("--- #yellow#image: %s v%s#reset#" % (image.name, image.version))
if args.all_version:
......@@ -183,7 +182,7 @@ def c_chroot(args):
'''
Helper to go cleanly inside a chroot
'''
istools.chroot(args.path, shell=args.shell, mount=not args.no_mount)
chroot(args.path, shell=args.shell, mount=not args.no_mount)
def c_clean(args):
'''
......@@ -334,7 +333,7 @@ def c_install(args):
arrow(u"Installing %s v%s" % (image.name, image.version))
# let's go
dt = image.run(args.parser, subparser, run_setup=not args.dry_run)
arrow(u"Install time: %s" % datetime.timedelta(seconds=dt))
arrow(u"Install time: %s" % timedelta(seconds=dt))
def c_list(args):
'''
......@@ -421,7 +420,7 @@ def c_prepare_chroot(args):
'''
Helper to prepare a path to be chrooted
'''
istools.prepare_chroot(args.path, mount=not args.no_mount)
prepare_chroot(args.path, mount=not args.no_mount)
def c_repo(args):
'''
......@@ -451,7 +450,7 @@ def c_unprepare_chroot(args):
'''
Helper to remove chroot preparation of a path
'''
istools.unprepare_chroot(args.path, mount=not args.no_umount)
unprepare_chroot(args.path, mount=not args.no_umount)
def c_upgrade(args):
'''
......@@ -464,16 +463,16 @@ def c_version(args):
'''
Display installsystems version
'''
out(installsystems.version)
out(VERSION)
def arg_parser_init():
'''
Create command parser
'''
# top level argument parsing
parser = argparse.ArgumentParser()
parser = ArgumentParser()
parser.add_argument("-V", "--version", action="version",
version=installsystems.version)
version=VERSION)
# exclusive group on verbosity
g = parser.add_mutually_exclusive_group()
g.add_argument("-v", "--verbosity", default=1,
......@@ -761,29 +760,26 @@ def main():
Program main
'''
try:
# by default full debug
setmode(2)
# init arg parser
arg_parser = arg_parser_init()
# encode command line arguments to utf-8
args = istools.argv()[1:]
args = argv()[1:]
# first partial parsing, to get early debug and config path
options = arg_parser.parse_known_args(args=args)[0]
# set early command line verbosity and color
installsystems.verbosity = options.verbosity
installsystems.printer.NOCOLOR = options.no_color
setmode(options.verbosity, options.no_color)
# load main config file options
config_parser = MainConfigFile(options.config, "installsystems")
options = config_parser.parse()
# second partial parsing, command line option overwrite config file
options = arg_parser.parse_known_args(args=args, namespace=options)[0]
# set verbosity and color
installsystems.verbosity = options.verbosity
installsystems.printer.NOCOLOR = options.no_color
# no warning if we are not in debug mode
if installsystems.verbosity < 2:
warnings.filterwarnings("ignore")
setmode(options.verbosity, options.no_color)
# nice and ionice process
if options.nice is not None or options.ionice_class is not None:
proc = psutil.Process(os.getpid())
proc = Process(getpid())
if options.nice is not None:
try:
proc.nice = options.nice
......@@ -793,10 +789,10 @@ def main():
if options.ionice_class is not None:
try:
ioclassmap = {
"none": psutil.IOPRIO_CLASS_NONE,
"rt": psutil.IOPRIO_CLASS_RT,
"be": psutil.IOPRIO_CLASS_BE,
"idle": psutil.IOPRIO_CLASS_IDLE}
"none": IOPRIO_CLASS_NONE,
"rt": IOPRIO_CLASS_RT,
"be": IOPRIO_CLASS_BE,
"idle": IOPRIO_CLASS_IDLE}
proc.set_ionice(ioclassmap[options.ionice_class], options.ionice_level)
debug(u"Setting ionice to class %s, level %s" %
(options.ionice_class, options.ionice_level))
......@@ -804,7 +800,7 @@ def main():
warn(u"Unable to ionice process to %s" % options.ionice_class)
# set timeout option
if options.timeout is not None:
socket.setdefaulttimeout(options.timeout)
setdefaulttimeout(options.timeout)
debug("Global timeout setted to %ds" % options.timeout)
# except for install command we parse all args!
# install command is responsible of parsing
......@@ -813,15 +809,15 @@ def main():
# let's go
options.func(options)
exit(0)
except UnicodeDecodeError as e:
except UnicodeDecodeError:
error("Unable to decode some characters. Check your locale settings.")
except KeyboardInterrupt:
warn("Keyboard Interrupted")
exit(1)
except ISError as e:
error(exception=e)
except Exception as e:
error(u"Unexpected error, please report it with debug enabled", exception=e)
except ISException as err:
error(exception=err)
except Exception as err:
error(u"Unexpected error, please report it with debug enabled", exception=err)
# Entry point
......
......@@ -17,33 +17,36 @@
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
InstallSystems module
InstallSystems package
'''
canonical_name="installsystems"
version = "@VERSION@"
verbosity = 1 # 0: quiet, 1: normal, 2: debug
def git_version():
import os
import sys
'''
Retrieve current git version
'''
from os import getcwd, chdir, devnull
from os.path import dirname
from subprocess import check_output, CalledProcessError
from sys import argv
version = ""
cwd = os.getcwd()
cwd = getcwd()
try:
os.chdir(os.path.dirname(sys.argv[0]))
chdir(dirname(argv[0]))
version = check_output(["git", "describe", "--tags", "--always" ],
stdin=open(os.devnull, 'rb'),
stderr=open(os.devnull, "wb")).strip()
stdin=open(devnull, 'rb'),
stderr=open(devnull, "wb")).strip()
if len(version) > 0:
version = "-" + version
except (OSError, CalledProcessError):
pass
finally:
os.chdir(cwd)
chdir(cwd)
return version
if version.find("+git") >= 0:
version += git_version()
CANONICAL_NAME = "@PACKAGE@"
VERSION = "@VERSION@"
if VERSION.find("+git") >= 0:
VERSION += git_version()
__all__ = []
......@@ -17,53 +17,18 @@
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
InstallSystems Configuration files class
InstallSystems configuration files module
'''
import codecs
import os
import sys
from argparse import Namespace
from configobj import ConfigObj, flatten_errors
from installsystems.exception import ISWarning, ISError
from installsystems.printer import warn, debug
from installsystems.repository.config import RepositoryConfig
from os import access, mkdir, getuid, R_OK, W_OK, X_OK
from os.path import join, expanduser, isfile, basename, abspath, exists, isdir
from sys import argv
from validate import Validator
from installsystems.exception import *
from installsystems.printer import *
from installsystems.repository import RepositoryConfig
# This must not be an unicode string, because configobj don't decode configspec
# with the provided encoding
MAIN_CONFIG_SPEC = '''\
[installsystems]
verbosity = integer(0, 2)
repo_config = string
repo_search = string
repo_filter = string
repo_timeout = integer
cache = string(default=%s)
timeout = integer
no_cache = boolean
no_check = boolean
no-sync = boolean
no_color = boolean
nice = integer
ionice_class = option("none", "rt", "be", "idle")
ionice_level = integer
'''
# This must not be an unicode string, because configobj don't decode configspec
# with the provided encoding
REPO_CONFIG_SPEC = '''\
[__many__]
path = string
fmod = string
dmod = string
uid = string
gid = string
offline = boolean
lastpath = string
dbpath = string
'''
class ConfigFile(object):
'''
......@@ -75,8 +40,8 @@ class ConfigFile(object):
Filename can be full path to config file or a name in config directory
'''
# try to get filename in default config dir
if os.path.isfile(filename):
self.path = os.path.abspath(filename)
if isfile(filename):
self.path = abspath(filename)
else:
self.path = self._config_path(filename)
# loading config file if exists
......@@ -104,53 +69,58 @@ class ConfigFile(object):
# remove wrong value to avoid merging it with argparse value
del self.config[section[0]][optname]
def _config_path(self, name):
@staticmethod
def _config_path(name):
'''
Return path of the best config file
'''
for cf in [ os.path.join(os.path.expanduser(u"~/.config/installsystems/%s.conf" % name)),
u"/etc/installsystems/%s.conf" % name ]:
if (os.path.isfile(cf) and os.access(cf, os.R_OK)):
return cf
for cfp in [join(expanduser(u"~/.config/installsystems/%s.conf" %name)),
u"/etc/installsystems/%s.conf" % name ]:
if (isfile(cfp) and access(cfp, R_OK)):
return cfp
return None
@property
def configspec(self):
'''Return configobj spec'''
raise NotImplementedError()
class MainConfigFile(ConfigFile):
'''
Program configuration class
'''
def __init__(self, filename, prefix=os.path.basename(sys.argv[0])):
def __init__(self, filename, prefix=basename(argv[0])):
self.prefix = prefix
self.configspec = (MAIN_CONFIG_SPEC % self.cache).splitlines()
try:
super(MainConfigFile, self).__init__(filename)
debug(u"Loading main config file: %s" % self.path)
except ISWarning:
debug("No main config file to load")
except Exception as e:
raise ISError(u"Unable load main config file %s" % self.path, e)
except Exception as exc:
raise ISError(u"Unable load main config file %s" % self.path, exc)
def _cache_paths(self):
'''
List all candidates to cache directories. Alive or not
'''
dirs = [os.path.expanduser("~/.cache"), "/var/tmp", "/tmp"]
dirs = [expanduser("~/.cache"), "/var/tmp", "/tmp"]
# we have an additional directory if we are root
if os.getuid() == 0:
if getuid() == 0:
dirs.insert(0, "/var/cache")
return map(lambda x: os.path.join(x, self.prefix), dirs)
return [ join(x, self.prefix) for x in dirs ]
def _cache_path(self):
'''
Return path of the best cache directory
'''
# find a good directory
for di in self._cache_paths():
if (os.path.exists(di)
and os.path.isdir(di)
and os.access(di, os.R_OK|os.W_OK|os.X_OK)):
return di
for directory in self._cache_paths():
if (exists(directory)
and isdir(directory)
and access(directory, R_OK|W_OK|X_OK)):
return directory
return None
@property
......@@ -159,12 +129,12 @@ class MainConfigFile(ConfigFile):
Find a cache directory
'''
if self._cache_path() is None:
for di in self._cache_paths():
for directory in self._cache_paths():
try:
os.mkdir(di)
mkdir(directory)
break
except Exception as e:
debug(u"Unable to create %s: %s" % (di, e))
except Exception as exc:
debug(u"Unable to create %s: %s" % (directory, exc))
return self._cache_path()
def parse(self, namespace=None):
......@@ -178,6 +148,11 @@ class MainConfigFile(ConfigFile):
setattr(namespace, option, value)
return namespace
@property
def configspec(self):
'''Return configobj spec'''
return (MAIN_CONFIG_SPEC % self.cache).splitlines()
class RepoConfigFile(ConfigFile):
'''
......@@ -188,7 +163,7 @@ class RepoConfigFile(ConfigFile):
# seting default config
self._config = {}
self._repos = []
self.configspec = REPO_CONFIG_SPEC.splitlines()
try:
super(RepoConfigFile, self).__init__(filename)
debug(u"Loading repository config file: %s" % self.path)
......@@ -207,9 +182,11 @@ class RepoConfigFile(ConfigFile):
if "path" not in self.config[rep]:
continue
# get all options in repo
self._repos.append(RepositoryConfig(rep, **dict(self.config[rep].items())))
except Exception as e:
raise ISError(u"Unable to load repository file %s" % self.path, e)
self._repos.append(
RepositoryConfig(rep, **dict(self.config[rep].items()))
)
except Exception as exc:
raise ISError(u"Unable to load repository file %s" % self.path, exc)
@property
def repos(self):
......@@ -218,3 +195,43 @@ class RepoConfigFile(ConfigFile):
'''
# deep copy
return list(self._repos)
@property
def configspec(self):
'''Return configobj spec'''
return REPO_CONFIG_SPEC.splitlines()
# This must not be an unicode string, because configobj don't decode configspec
# with the provided encoding
MAIN_CONFIG_SPEC = '''\
[installsystems]
verbosity = integer(0, 2)
repo_config = string
repo_search = string
repo_filter = string
repo_timeout = integer
cache = string(default=%s)
timeout = integer
no_cache = boolean
no_check = boolean
no-sync = boolean
no_color = boolean
nice = integer
ionice_class = option("none", "rt", "be", "idle")
ionice_level = integer
'''
# This must not be an unicode string, because configobj don't decode configspec
# with the provided encoding
REPO_CONFIG_SPEC = '''\
[__many__]
path = string
fmod = string
dmod = string
uid = string
gid = string
offline = boolean
lastpath = string
dbpath = string
'''
# -*- python -*-
# -*- coding: utf-8 -*-
# Installsystems - Python installation framework
# Copyright © 2011-2012 Smartjog S.A
# Copyright © 2011-2012 Sébastien Luttringer
# This file is part of Installsystems.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# Installsystems is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# Installsystems is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# You should have received a copy of the GNU Lesser General Public License
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
InstallSystems Exceptions
'''
import traceback
import sys
from traceback import print_tb, print_exc, format_exception_only
from sys import exc_info, stderr
class ISException(Exception):
'''
......@@ -32,8 +29,9 @@ class ISException(Exception):
'''
def __init__(self, message=u"", exception=None):
Exception.__init__(self)
self.message = unicode(message)
self.exception = None if exception is None else sys.exc_info()
self.exception = None if exception is None else exc_info()
def __str__(self):
'''
......@@ -44,7 +42,7 @@ class ISException(Exception):
else:
return self.message
def print_sub_tb(self, fd=sys.stderr):
def print_sub_tb(self, fd=stderr):
'''
Print stored exception traceback and exception message
'''
......@@ -53,20 +51,21 @@ class ISException(Exception):
return
# print traceback and exception separatly to avoid recursive print of
# "Traceback (most recent call last)" from traceback.print_exception
traceback.print_tb(self.exception[2], file=fd)
fd.write("".join(traceback.format_exception_only(self.exception[0], self.exception[1])))
print_tb(self.exception[2], file=fd)
fd.write("".join(format_exception_only(self.exception[0],
self.exception[1])))
# recursively call traceback print on ISException error
if isinstance(self.exception[1], ISException):
self.exception[1].print_sub_tb()
def print_tb(self, fd=sys.stderr):
def print_tb(self, fd=stderr):
'''
Print traceback from embeded exception or current one
'''
from installsystems.printer import out
# coloring
out("#l##B#", fd=fd, endl="")
traceback.print_exc(file=fd)
print_exc(file=fd)
self.print_sub_tb(fd)
# reset color
out("#R#", fd=fd, endl="")
......@@ -89,5 +88,5 @@ class InvalidSourceImage(ISError):
Invalid source image errors
'''
def __init(self, message=u"", exception=None):
ISException(self, u"Invalid source image: " + message, exception)
def __init__(self, message=u"", exception=None):
ISError.__init__(self, u"Invalid source image: " + message, exception)
# -*- python -*-
# -*- coding: utf-8 -*-
# This file is part of Installsystems.
#
# Installsystems is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Installsystems is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
InstallSystems image package
'''
from installsystems.image.image import Image
from installsystems.image.source import SourceImage
from installsystems.image.package import PackageImage
from installsystems.image.payload import Payload
from installsystems.image.changelog import Changelog
# -*- python -*-
# -*- coding: utf-8 -*-
# This file is part of Installsystems.
#
# Installsystems is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Installsystems is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
Image Changelog module
'''
from installsystems.exception import ISError
from installsystems.printer import out
from installsystems.tools import strvercmp
from os import linesep
from re import match