Commit 26aaa96f authored by Seblu's avatar Seblu
Browse files

Improve remote cache. Http transport enabled

parent 1a56a5d0

bin/isinstall

+2 −2
Original line number Diff line number Diff line
@@ -45,9 +45,9 @@ try: 
    # load config

    config = ConfigFile("isinstall", args.config)

    # looks if arguments is a file or image name

    image_name_type = istools.get_path_type(args.image_name)

    image_name_type = istools.pathtype(args.image_name)

    if image_name_type == "file":

        pkg = PackageImage(istools.complete_path(args.image_name))

        pkg = PackageImage(istools.abspath(args.image_name))

    elif image_name_type == "name":

        # init repo cache object

        repocache = RepositoryCache(config.cache, verbose=args.verbose)

installsystems/config.py

+5 −4
Original line number Diff line number Diff line
@@ -32,7 +32,7 @@ class ConfigFile(object): 
        '''Load/Reload config file'''

        # seting default config

        self._config = {}

        self._repos = {}

        self._repos = []

        # loading config file if exists

        if self.path is not None:

            debug("Loading config file: %s" % self.path)
@@ -49,7 +49,7 @@ class ConfigFile(object): 
                    if "image" not in cp.options(rep):

                        continue

                    # get all options in repo

                    self._repos[rep] = RepositoryConfig(rep, **dict(cp.items(rep)))

                    self._repos.append( RepositoryConfig(rep, **dict(cp.items(rep))))

            except Exception as e:

                raise

                raise Exception("Unable load file %s: %s" % (self.path, e))
@@ -89,5 +89,6 @@ class ConfigFile(object): 


    @property

    def repos(self):

        '''Get a list of repository available'''

        return self._repos.copy()
 
        '''Get a dict of repository available'''

        # deep copy

        return list(self._repos)

installsystems/database.py

+2 −2
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ class Database(object): 
    @classmethod

    def create(cls, path, verbose=True):

        arrow("Creating repository database", 1, verbose)

        dbpath = os.path.abspath(path)

        dbpath = istools.abspath(path)

        if os.path.exists(dbpath):

            raise Exception("db already exists")

        try:
@@ -34,7 +34,7 @@ class Database(object): 
        return cls(path, verbose)



    def __init__(self, path, verbose=True):

        self.path = os.path.abspath(path)

        self.path = istools.abspath(path)

        self.verbose = verbose



    def add(self, package):

installsystems/repository.py

+51 −36
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ import time 
import shutil

import pwd

import grp

import copy

import installsystems

import installsystems.tools as istools

from installsystems.printer import *
@@ -176,61 +177,75 @@ class RepositoryCache(object): 
    '''Local repository cache class'''



    def __init__(self, cache_path, verbose=True):

        self.base_path = os.path.abspath(cache_path)

        self.image_path = os.path.join(self.base_path, "image")

        self.last_path = os.path.join(self.base_path, "last")

        self.db_path = os.path.join(self.base_path, "db")

        for path in (self.base_path, self.image_path, self.last_path, self.db_path):

            if not os.path.exists(path):

                os.mkdir(path)

            if not os.access(path, os.W_OK | os.X_OK):

                raise Exception("%s is not writable or executable" % path)

        self.verbose = verbose

        self.repos = dict()

        debug("Repository cache is in %s" % self.base_path)

        self.repos = {}

        self.path = os.path.abspath(cache_path)

        # ensure cache directories are avaiblable

        if not os.path.exists(self.path):

            os.mkdir(self.path)

        if not os.access(self.path, os.W_OK | os.X_OK):

            raise Exception("%s is not writable or executable" % path)

        debug("Repository cache is in %s" % self.path)



    def register(self, iterepo):

        '''Register a repository to track'''

        for r in iterepo:

            self.repos[r[0]] = Repository(istools.abspath(r[1]),

                                          istools.abstpath(r[2]),

                                          verbose=self.verbose)

    def register(self, configs):

        '''Register a list of repository from its config'''

        for conf in configs:

            self.repos[conf.name] = {}

            # keep original repository conf

            self.repos[conf.name]["orig"] = Repository(conf, self.verbose)

            # change configuration to make remote repository in cache

            cconf = copy.copy(conf)

            cconf.image = os.path.join(self.path, conf.name)

            cconf.data = "/dev/null"

            self.repos[conf.name]["cache"] = Repository(cconf, self.verbose)

            # create a local directory

            if not os.path.exists(cconf.image):

                os.mkdir(cconf.image)



    def update(self):

        '''Update cache info'''

        arrow("Updating repositories", 1, self.verbose)

        for r in self.repos:

            debug("%s: remote_last: %s, local_last:%s" % (r,

                                                          self.repos[r].last(),

                                                          self.last(r)))

            if self.repos[r].last() > self.last(r):

            # last local

            local_last = self.last(r)

            # copy last file

                istools.copy(self.repos[r].last_path, os.path.join(self.last_path, r))

            arrow("Copying %s repository last" % r, 2, self.verbose)

            istools.copy(self.repos[r]["orig"].last_path,

                         self.repos[r]["cache"].last_path,)

            # last after update

            remote_last = self.last(r)

            debug("%s: last: local: %s, remote:%s" % (r, local_last, remote_last))

            # Updating db?

            remote_db = self.repos[r]["orig"].db.path

            local_db = self.repos[r]["cache"].db.path

            if remote_last > local_last or not os.path.exists(local_db):

                # copy db file

                istools.copy(self.repos[r].db.path, os.path.join(self.db_path, r))

                arrow("Copying %s repository db" % r, 2, self.verbose)

                istools.copy(remote_db, local_db)

                arrow("%s updated" % r, 2, self.verbose)



    def last(self, reponame):

        '''Return the last timestamp of a repo'''

        last_path = os.path.join(self.last_path, reponame)

        if os.path.exists(last_path):

        last_path = os.path.join(self.path, reponame, "last")

        try:

            return int(open(last_path, "r").read().rstrip())

        except Exception:

            return 0



    def get_image(self, reponame, imagename, imageversion):

        '''Obtain a local path in cache for a remote image in repo'''

        arrow("Getting image", 1, self.verbose)

        filename = "%s-%s%s" % (imagename, imageversion, Image.image_extension)

        localpath = os.path.join(self.image_path, filename)

        cachepath = os.path.join(self.repos[reponame]["cache"].config.image, filename)

        # return db path if exists

        if os.path.exists(localpath):

        if os.path.exists(cachepath):

            arrow("Found in cache", 2, self.verbose)

            return localpath

        else:

            # get remote image

        remotepath = os.path.join(self.repos[reponame].image_path, filename)

            remotepath = os.path.join(self.repos[reponame]["orig"].config.image, filename)

            arrow("Copying from repository", 2, self.verbose)

        istools.copy(remotepath, localpath)

        return localpath

            istools.copy(remotepath, cachepath)

        return cachepath



    def find_image(self, name, version):

        '''Find an image in repositories'''
@@ -241,7 +256,7 @@ class RepositoryCache(object): 
        img = None

        # search in all repositories

        for repo in self.repos:

            tempdb = Database(os.path.join(self.db_path, repo), False)

            tempdb = Database(self.repos[repo]["cache"].db.path, False)

            img = tempdb.find(name, version)

            if img is not None:

                # \o/

installsystems/tools.py

+10 −3
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ InstallSystems Generic Tools Library 
import os

import hashlib

import shutil

import urllib2



def md5sum(path):

    '''Compute md5 of a file'''
@@ -16,20 +17,26 @@ def md5sum(path): 
    m.update(open(path, "r").read())

    return m.hexdigest()



def copy(source, destination, uid=None, gid=None, mode=None):

def copy(source, destination, uid=None, gid=None, mode=None, timeout=None):

    '''Copy a source to destination. Take care of path type'''

    stype = pathtype(source)

    dtype = pathtype(destination)

    # ensure destination is not a directory

    if dtype == "file" and os.path.isdir(destination):

        destination = os.path.join(destination, os.path.basename(source))

    # trivial case

    if stype == dtype == "file":

        shutil.copy(source, destination)

    elif stype == "http" and dtype == "file":

        f_dest = open(destination, "w")

        f_source = urllib2.urlopen(source, timeout=timeout)

        f_dest.write(f_source.read())

    elif stype == "file" and dtype == "":

        raise NotImplementedError

    else:

        raise NotImplementedError

    # setting destination file rights

    if dtype == "file":

        if os.path.isdir(destination):

            destination = os.path.join(destination, os.path.basename(source))

        chrights(destination, uid, gid, mode)



def mkdir(path, uid=None, gid=None, mode=None):