Commit 084ed9b5 authored by Sébastien Luttringer's avatar Sébastien Luttringer
Browse files

Deep refactoring 

- 2 new packages (image and repository).
- Split big modules into little one under packages.
- Replace full module import by needed objects
- Avoid wildcard imports
- Fix a lot of pylint error and warnings on modules
parent 118f3047

Makefile.am

+22 −5
Original line number Diff line number Diff line
@@ -28,20 +28,37 @@ dist_doc_DATA = README COPYRIGHT LICENSE AUTHORS DEPENDENCIES 
dist_bin_SCRIPTS = bin/is

noinst_SCRIPTS = installsystems/__init__.py



# python library

# python installsystems package

installsystemsdir=$(pythondir)/installsystems

installsystems_PYTHON = \

	installsystems/__init__.py \

	installsystems/config.py \

	installsystems/database.py \

	installsystems/exception.py \

	installsystems/image.py \

	installsystems/printer.py \

	installsystems/repository.py \

	installsystems/tarball.py \

	installsystems/template.py \

	installsystems/tools.py



installsystemsimagedir=$(pythondir)/installsystems/image

installsystemsimage_PYTHON = \

	installsystems/image/__init__.py \

	installsystems/image/changelog.py \

	installsystems/image/image.py \

	installsystems/image/package.py \

	installsystems/image/payload.py \

	installsystems/image/source.py \

	installsystems/image/tarball.py



installsystemsrepositorydir=$(pythondir)/installsystems/repository

installsystemsrepository_PYTHON = \

	installsystems/repository/__init__.py \

	installsystems/repository/config.py \

	installsystems/repository/database.py \

	installsystems/repository/factory.py \

	installsystems/repository/manager.py \

	installsystems/repository/repository.py \

	installsystems/repository/repository1.py \

	installsystems/repository/repository2.py



do_substitution = $(SED) -e 's,[@]pythondir[@],$(pythondir),g' \

	-e 's,[@]PACKAGE[@],$(PACKAGE),g' \

	-e 's,[@]VERSION[@],$(VERSION),g'

bin/is

+51 −55
Original line number Diff line number Diff line
@@ -20,25 +20,24 @@ 
InstallSystems Command line Tool

'''



import os

import datetime

import re

import fnmatch

import warnings

import argparse

import psutil

import socket

import sys

import installsystems

import installsystems.printer

import installsystems.tools as istools

from installsystems.exception import *

from installsystems.printer import *

from installsystems.repository import Repository

from installsystems.repository import RepositoryManager

from installsystems.repository import RepositoryConfig

from installsystems.image import PackageImage, SourceImage

from argparse import ArgumentParser

from datetime import timedelta

from installsystems import VERSION

from installsystems.config import MainConfigFile, RepoConfigFile

from installsystems.exception import ISError, ISException

from installsystems.image import PackageImage, SourceImage

from installsystems.printer import arrow, arrowlevel, setmode

from installsystems.printer import out, warn, error, debug, confirm

from installsystems.repository import Repository, RepositoryManager, RepositoryConfig

from installsystems.tools import chroot, prepare_chroot, unprepare_chroot

from installsystems.tools import isfile, smd5sum, argv

from os import getpid, getcwdu, chdir

from psutil import IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE

from psutil import Process, IOPRIO_CLASS_NONE

from socket import setdefaulttimeout



# used by os.path.isfile

import os



################################################################################

# Common functions
@@ -52,15 +51,15 @@ def load_repositories(args): 
    if args.no_cache:

        args.cache = None

    # split filter and search in list

    args.repo_filter = Repository.split_repository_list(args.repo_filter)

    args.repo_search = Repository.split_repository_list(args.repo_search)

    args.repo_filter = Repository.split_list(args.repo_filter)

    args.repo_search = Repository.split_list(args.repo_search)

    # init repo cache object

    repoman = RepositoryManager(args.cache, timeout=args.repo_timeout or args.timeout,

                                filter=args.repo_filter, search=args.repo_search)

    # register repositories (order matter)

    # load repo configs from command line

    if args.repo_path != "":

        repoconf = RepositoryConfig(istools.smd5sum(args.repo_path)[:8],

        repoconf = RepositoryConfig(smd5sum(args.repo_path)[:8],

                                    path=args.repo_path)

        repoman.register(repoconf, temp=True, nosync=args.no_sync)

    # load repo configs from config
@@ -80,7 +79,7 @@ def get_images(patterns, repoman, local=True, min=None, max=None): 
    ans = []

    for pattern in patterns:

        # check if image is a local file

        if local and istools.isfile(pattern) and os.path.isfile(pattern):

        if local and isfile(pattern) and os.path.isfile(pattern):

            ans.append((pattern, None))

        else: # we need to find image in a repository

            ans += sorted(repoman.select_images([pattern]).items())
@@ -128,8 +127,8 @@ def c_build(args): 
        arrow("Build %s" % path)

        # chdir inside path if --chdir

        if args.chdir:

            cwd = os.getcwdu()

            os.chdir(path)

            cwd = getcwdu()

            chdir(path)

            path = "."

        arrowlevel(1)

        # load source image
@@ -138,19 +137,19 @@ def c_build(args): 
        dt = simg.build(force=args.force, force_payload=args.payload,

                        check=not args.no_check, script=not args.no_script)

        gdt += dt

        arrow(u"Build time: %s" % datetime.timedelta(seconds=dt))

        arrow(u"Build time: %s" % timedelta(seconds=dt))

        if args.chdir:

            os.chdir(cwd)

            chdir(cwd)

        arrowlevel(-1)

    if len(args.paths) > 1:

        arrow(u"Global build time: %s" % datetime.timedelta(seconds=gdt))

        arrow(u"Global build time: %s" % timedelta(seconds=gdt))



def c_cat(args):

    '''

    Display files inside a packaged image

    '''

    repoman = load_repositories(args)

    image, repo = next(get_images([args.pattern], repoman, min=1, max=1))

    image = next(get_images([args.pattern], repoman, min=1, max=1))[0]

    for filename in args.file:

        image.cat(filename)
@@ -160,7 +159,7 @@ def c_changelog(args): 
    '''

    repoman = load_repositories(args)

    images = list(get_images(args.pattern, repoman, min=1))

    for image, repo in images:

    for image in images:

        if len(images) > 1:

            out("--- #yellow#image: %s v%s#reset#" % (image.name, image.version))

        if args.all_version:
@@ -183,7 +182,7 @@ def c_chroot(args): 
    '''

    Helper to go cleanly inside a chroot

    '''

    istools.chroot(args.path, shell=args.shell, mount=not args.no_mount)

    chroot(args.path, shell=args.shell, mount=not args.no_mount)



def c_clean(args):

    '''
@@ -334,7 +333,7 @@ def c_install(args): 
    arrow(u"Installing %s v%s" % (image.name, image.version))

    # let's go

    dt = image.run(args.parser, subparser, run_setup=not args.dry_run)

    arrow(u"Install time: %s" % datetime.timedelta(seconds=dt))

    arrow(u"Install time: %s" % timedelta(seconds=dt))



def c_list(args):

    '''
@@ -421,7 +420,7 @@ def c_prepare_chroot(args): 
    '''

    Helper to prepare a path to be chrooted

    '''

    istools.prepare_chroot(args.path, mount=not args.no_mount)

    prepare_chroot(args.path, mount=not args.no_mount)



def c_repo(args):

    '''
@@ -451,7 +450,7 @@ def c_unprepare_chroot(args): 
    '''

    Helper to remove chroot preparation of a path

    '''

    istools.unprepare_chroot(args.path, mount=not args.no_umount)

    unprepare_chroot(args.path, mount=not args.no_umount)



def c_upgrade(args):

    '''
@@ -464,16 +463,16 @@ def c_version(args): 
    '''

    Display installsystems version

    '''

    out(installsystems.version)

    out(VERSION)



def arg_parser_init():

    '''

    Create command parser

    '''

    # top level argument parsing

    parser = argparse.ArgumentParser()

    parser = ArgumentParser()

    parser.add_argument("-V", "--version", action="version",

                        version=installsystems.version)

                        version=VERSION)

    # exclusive group on verbosity

    g = parser.add_mutually_exclusive_group()

    g.add_argument("-v", "--verbosity", default=1,
@@ -761,29 +760,26 @@ def main(): 
    Program main

    '''

    try:

        # by default full debug

        setmode(2)

        # init arg parser

        arg_parser = arg_parser_init()

        # encode command line arguments to utf-8

        args = istools.argv()[1:]

        args = argv()[1:]

        # first partial parsing, to get early debug and config path

        options = arg_parser.parse_known_args(args=args)[0]

        # set early command line verbosity and color

        installsystems.verbosity = options.verbosity

        installsystems.printer.NOCOLOR = options.no_color

        setmode(options.verbosity, options.no_color)

        # load main config file options

        config_parser = MainConfigFile(options.config, "installsystems")

        options = config_parser.parse()

        # second partial parsing, command line option overwrite config file

        options = arg_parser.parse_known_args(args=args, namespace=options)[0]

        # set verbosity and color

        installsystems.verbosity = options.verbosity

        installsystems.printer.NOCOLOR = options.no_color

        # no warning if we are not in debug mode

        if installsystems.verbosity < 2:

            warnings.filterwarnings("ignore")

        setmode(options.verbosity, options.no_color)

        # nice and ionice process

        if options.nice is not None or options.ionice_class is not None:

            proc = psutil.Process(os.getpid())

            proc = Process(getpid())

            if options.nice is not None:

                try:

                    proc.nice = options.nice
@@ -793,10 +789,10 @@ def main(): 
            if options.ionice_class is not None:

                try:

                    ioclassmap = {

                        "none": psutil.IOPRIO_CLASS_NONE,

                        "rt": psutil.IOPRIO_CLASS_RT,

                        "be": psutil.IOPRIO_CLASS_BE,

                        "idle": psutil.IOPRIO_CLASS_IDLE}

                        "none": IOPRIO_CLASS_NONE,

                        "rt": IOPRIO_CLASS_RT,

                        "be": IOPRIO_CLASS_BE,

                        "idle": IOPRIO_CLASS_IDLE}

                    proc.set_ionice(ioclassmap[options.ionice_class], options.ionice_level)

                    debug(u"Setting ionice to class %s, level %s" %

                          (options.ionice_class, options.ionice_level))
@@ -804,7 +800,7 @@ def main(): 
                    warn(u"Unable to ionice process to %s" % options.ionice_class)

        # set timeout option

        if options.timeout is not None:

            socket.setdefaulttimeout(options.timeout)

            setdefaulttimeout(options.timeout)

            debug("Global timeout setted to %ds" % options.timeout)

        # except for install command we parse all args!

        # install command is responsible of parsing
@@ -813,15 +809,15 @@ def main(): 
        # let's go

        options.func(options)

        exit(0)

    except UnicodeDecodeError as e:

    except UnicodeDecodeError:

        error("Unable to decode some characters. Check your locale settings.")

    except KeyboardInterrupt:

        warn("Keyboard Interrupted")

        exit(1)

    except ISError as e:

        error(exception=e)

    except Exception as e:

        error(u"Unexpected error, please report it with debug enabled", exception=e)

    except ISException as err:

        error(exception=err)

    except Exception as err:

        error(u"Unexpected error, please report it with debug enabled", exception=err)





# Entry point

installsystems/__init__.py.in

+17 −14
Original line number Diff line number Diff line
@@ -17,33 +17,36 @@ 
# along with Installsystems.  If not, see <http://www.gnu.org/licenses/>.



'''

InstallSystems module

InstallSystems package

'''



canonical_name="installsystems"

version = "@VERSION@"

verbosity = 1 # 0: quiet, 1: normal, 2: debug



def git_version():

    import os

    import sys

    '''

    Retrieve current git version

    '''

    from os import getcwd, chdir, devnull

    from os.path import dirname

    from subprocess import check_output, CalledProcessError

    from sys import argv

    version = ""

    cwd = os.getcwd()

    cwd = getcwd()

    try:

        os.chdir(os.path.dirname(sys.argv[0]))

        chdir(dirname(argv[0]))

        version = check_output(["git", "describe", "--tags", "--always" ],

                               stdin=open(os.devnull, 'rb'),

                               stderr=open(os.devnull, "wb")).strip()

                               stdin=open(devnull, 'rb'),

                               stderr=open(devnull, "wb")).strip()

        if len(version) > 0:

            version = "-" + version

    except (OSError, CalledProcessError):

        pass

    finally:

        os.chdir(cwd)

        chdir(cwd)

    return version



if version.find("+git") >= 0:

    version += git_version()

CANONICAL_NAME = "@PACKAGE@"

VERSION = "@VERSION@"



if VERSION.find("+git") >= 0:

    VERSION += git_version()



__all__ = []

installsystems/config.py

+86 −69
Original line number Diff line number Diff line
@@ -17,53 +17,18 @@ 
# along with Installsystems.  If not, see <http://www.gnu.org/licenses/>.



'''

InstallSystems Configuration files class

InstallSystems configuration files module

'''



import codecs

import os

import sys

from argparse import Namespace

from configobj import ConfigObj, flatten_errors

from installsystems.exception import ISWarning, ISError

from installsystems.printer import warn, debug

from installsystems.repository.config import RepositoryConfig

from os import access, mkdir, getuid, R_OK, W_OK, X_OK

from os.path import join, expanduser, isfile, basename, abspath, exists, isdir

from sys import argv

from validate import Validator

from installsystems.exception import *

from installsystems.printer import *

from installsystems.repository import RepositoryConfig



# This must not be an unicode string, because configobj don't decode configspec

# with the provided encoding

MAIN_CONFIG_SPEC = '''\

[installsystems]

verbosity = integer(0, 2)

repo_config = string

repo_search = string

repo_filter = string

repo_timeout = integer

cache = string(default=%s)

timeout = integer

no_cache = boolean

no_check = boolean

no-sync = boolean

no_color = boolean

nice = integer

ionice_class = option("none", "rt", "be", "idle")

ionice_level = integer

'''



# This must not be an unicode string, because configobj don't decode configspec

# with the provided encoding

REPO_CONFIG_SPEC = '''\

[__many__]

    path = string

    fmod = string

    dmod = string

    uid = string

    gid = string

    offline = boolean

    lastpath = string

    dbpath = string

'''





class ConfigFile(object):

    '''
@@ -75,8 +40,8 @@ class ConfigFile(object): 
        Filename can be full path to config file or a name in config directory

        '''

        # try to get filename in default config dir

        if os.path.isfile(filename):

            self.path = os.path.abspath(filename)

        if isfile(filename):

            self.path = abspath(filename)

        else:

            self.path = self._config_path(filename)

        # loading config file if exists
@@ -104,53 +69,58 @@ class ConfigFile(object): 
                    # remove wrong value to avoid merging it with argparse value

                    del self.config[section[0]][optname]



    def _config_path(self, name):

    @staticmethod

    def _config_path(name):

        '''

        Return path of the best config file

        '''

        for cf in [ os.path.join(os.path.expanduser(u"~/.config/installsystems/%s.conf" % name)),

        for cfp in [join(expanduser(u"~/.config/installsystems/%s.conf" %name)),

         u"/etc/installsystems/%s.conf" % name ]:

            if (os.path.isfile(cf) and os.access(cf, os.R_OK)):

                return cf

            if (isfile(cfp) and access(cfp, R_OK)):

                return cfp

        return None



    @property

    def configspec(self):

        '''Return configobj spec'''

        raise NotImplementedError()





class MainConfigFile(ConfigFile):

    '''

    Program configuration class

    '''



    def __init__(self, filename, prefix=os.path.basename(sys.argv[0])):

    def __init__(self, filename, prefix=basename(argv[0])):

        self.prefix = prefix

        self.configspec = (MAIN_CONFIG_SPEC % self.cache).splitlines()

        try:

            super(MainConfigFile, self).__init__(filename)

            debug(u"Loading main config file: %s" % self.path)

        except ISWarning:

            debug("No main config file to load")

        except Exception as e:

            raise ISError(u"Unable load main config file %s" % self.path, e)

        except Exception as exc:

            raise ISError(u"Unable load main config file %s" % self.path, exc)



    def _cache_paths(self):

        '''

        List all candidates to cache directories. Alive or not

        '''

        dirs = [os.path.expanduser("~/.cache"), "/var/tmp", "/tmp"]

        dirs = [expanduser("~/.cache"), "/var/tmp", "/tmp"]

        # we have an additional directory if we are root

        if os.getuid() == 0:

        if getuid() == 0:

            dirs.insert(0, "/var/cache")

        return map(lambda x: os.path.join(x, self.prefix), dirs)

        return [ join(x, self.prefix) for x in dirs ]



    def _cache_path(self):

        '''

        Return path of the best cache directory

        '''

        # find a good directory

        for di in self._cache_paths():

            if (os.path.exists(di)

                and os.path.isdir(di)

                and os.access(di, os.R_OK|os.W_OK|os.X_OK)):

                return di

        for directory in self._cache_paths():

            if (exists(directory)

                and isdir(directory)

                and access(directory, R_OK|W_OK|X_OK)):

                return directory

        return None



    @property
@@ -159,12 +129,12 @@ class MainConfigFile(ConfigFile): 
        Find a cache directory

        '''

        if self._cache_path() is None:

            for di in self._cache_paths():

            for directory in self._cache_paths():

                try:

                    os.mkdir(di)

                    mkdir(directory)

                    break

                except Exception as e:

                    debug(u"Unable to create %s: %s" % (di, e))

                except Exception as exc:

                    debug(u"Unable to create %s: %s" % (directory, exc))

        return self._cache_path()



    def parse(self, namespace=None):
@@ -178,6 +148,11 @@ class MainConfigFile(ConfigFile): 
                setattr(namespace, option, value)

        return namespace



    @property

    def configspec(self):

        '''Return configobj spec'''

        return (MAIN_CONFIG_SPEC % self.cache).splitlines()





class RepoConfigFile(ConfigFile):

    '''
@@ -188,7 +163,7 @@ class RepoConfigFile(ConfigFile): 
        # seting default config

        self._config = {}

        self._repos = []

        self.configspec = REPO_CONFIG_SPEC.splitlines()



        try:

            super(RepoConfigFile, self).__init__(filename)

            debug(u"Loading repository config file: %s" % self.path)
@@ -207,9 +182,11 @@ class RepoConfigFile(ConfigFile): 
                if "path" not in self.config[rep]:

                    continue

                # get all options in repo

                self._repos.append(RepositoryConfig(rep, **dict(self.config[rep].items())))

        except Exception as e:

            raise ISError(u"Unable to load repository file %s" % self.path, e)

                self._repos.append(

                    RepositoryConfig(rep, **dict(self.config[rep].items()))

                )

        except Exception as exc:

            raise ISError(u"Unable to load repository file %s" % self.path, exc)



    @property

    def repos(self):
@@ -218,3 +195,43 @@ class RepoConfigFile(ConfigFile): 
        '''

        # deep copy

        return list(self._repos)



    @property

    def configspec(self):

        '''Return configobj spec'''

        return REPO_CONFIG_SPEC.splitlines()





# This must not be an unicode string, because configobj don't decode configspec

# with the provided encoding

MAIN_CONFIG_SPEC = '''\

[installsystems]

verbosity = integer(0, 2)

repo_config = string

repo_search = string

repo_filter = string

repo_timeout = integer

cache = string(default=%s)

timeout = integer

no_cache = boolean

no_check = boolean

no-sync = boolean

no_color = boolean

nice = integer

ionice_class = option("none", "rt", "be", "idle")

ionice_level = integer

'''



# This must not be an unicode string, because configobj don't decode configspec

# with the provided encoding

REPO_CONFIG_SPEC = '''\

[__many__]

    path = string

    fmod = string

    dmod = string

    uid = string

    gid = string

    offline = boolean

    lastpath = string

    dbpath = string

'''

installsystems/exception.py

+21 −22
Original line number Diff line number Diff line 
# -*- python -*-

# -*- coding: utf-8 -*-



# Installsystems - Python installation framework

# Copyright © 2011-2012 Smartjog S.A

# Copyright © 2011-2012 Sébastien Luttringer

# This file is part of Installsystems.

#

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; either version 2

# of the License, or (at your option) any later version.

# Installsystems is free software: you can redistribute it and/or modify

# it under the terms of the GNU Lesser General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# Installsystems is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# GNU Lesser General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

# You should have received a copy of the GNU Lesser General Public License

# along with Installsystems.  If not, see <http://www.gnu.org/licenses/>.



'''

InstallSystems Exceptions

'''



import traceback

import sys

from traceback import print_tb, print_exc, format_exception_only

from sys import exc_info, stderr



class ISException(Exception):

    '''
@@ -32,8 +29,9 @@ class ISException(Exception): 
    '''



    def __init__(self, message=u"", exception=None):

        Exception.__init__(self)

        self.message = unicode(message)

        self.exception = None if exception is None else sys.exc_info()

        self.exception = None if exception is None else exc_info()



    def __str__(self):

        '''
@@ -44,7 +42,7 @@ class ISException(Exception): 
        else:

            return self.message



    def print_sub_tb(self, fd=sys.stderr):

    def print_sub_tb(self, fd=stderr):

        '''

        Print stored exception traceback and exception message

        '''
@@ -53,20 +51,21 @@ class ISException(Exception): 
            return

        # print traceback and exception separatly to avoid recursive print of

        # "Traceback (most recent call last)" from traceback.print_exception

        traceback.print_tb(self.exception[2], file=fd)

        fd.write("".join(traceback.format_exception_only(self.exception[0], self.exception[1])))

        print_tb(self.exception[2], file=fd)

        fd.write("".join(format_exception_only(self.exception[0],

                                               self.exception[1])))

        # recursively call traceback print on ISException error

        if isinstance(self.exception[1], ISException):

            self.exception[1].print_sub_tb()



    def print_tb(self, fd=sys.stderr):

    def print_tb(self, fd=stderr):

        '''

        Print traceback from embeded exception or current one

        '''

        from installsystems.printer import out

        # coloring

        out("#l##B#", fd=fd, endl="")

        traceback.print_exc(file=fd)

        print_exc(file=fd)

        self.print_sub_tb(fd)

        # reset color

        out("#R#", fd=fd, endl="")
@@ -89,5 +88,5 @@ class InvalidSourceImage(ISError): 
    Invalid source image errors

    '''



    def __init(self, message=u"", exception=None):

        ISException(self, u"Invalid source image: " + message, exception)
 
    def __init__(self, message=u"", exception=None):

        ISError.__init__(self, u"Invalid source image: " + message, exception)