repository.py

# -*- python -*-
# -*- coding: utf-8 -*-
# Started 10/05/2011 by Seblu <seblu@seblu.net>

'''
Repository stuff
'''

import os
import time
import shutil
import pwd
import grp
import copy
import installsystems
import installsystems.tools as istools
from installsystems.printer import *
from installsystems.tarball import Tarball
from installsystems.image import Image, PackageImage
from installsystems.database import Database

class Repository(object):
    '''Repository class'''

    def __init__(self, config, verbose=True):
        self.verbose = verbose
        self.config = config
        self.db = Database(os.path.join(config.path, config.dbname), verbose=self.verbose)

    @classmethod
    def create(cls, config, verbose=True):
        '''Create an empty base repository'''
        # check local repository
        if istools.pathtype(config.path) != "file":
            raise NotImplementedError("Repository creation must be local")
        # create base directories
        arrow("Creating base directories", 1, verbose)
        # creating local directory
        try:
            if os.path.exists(config.path):
                arrow("%s already exists" % config.path, 2, verbose)
            else:
                istools.mkdir(config.path, config.chown, config.chgroup, config.dchmod)
                arrow("%s directory created" % config.path, 2, verbose)
        except Exception as e:
            raise Exception("Unable to create directory %s: %s" % (config.path, e))
        # create database
        dbpath = os.path.join(config.path, config.dbname)
        d = Database.create(dbpath, verbose=verbose)
        istools.chrights(dbpath, config.chown, config.chgroup, config.fchmod)
        # create last file
        arrow("Creating last file", 1, verbose)
        self = cls(config, verbose)
        self.update_last()
        return self

    def update_last(self):
        '''Update last file to current time'''
        # check local repository
        if istools.pathtype(self.config.path) != "file":
            raise NotImplementedError("Repository addition must be local")
        try:
            arrow("Updating last file", 1, self.verbose)
            last_path = os.path.join(self.config.path, self.config.lastname)
            open(last_path, "w").write("%s\n" % int(time.time()))
            os.chown(last_path, self.config.chown, self.config.chgroup)
            os.chmod(last_path, self.config.fchmod)
        except Exception as e:
            raise Exception("Update last file failed: %s" % e)

    def last(self):
        '''Return the last value'''
        try:
            last_path = os.path.join(config.path, config.lastname)
            return int(istools.uopen(last_path, "r").read().rstrip())
        except Exception as e:
            raise Exception("Read last file failed: %s" % e)
        return 0

    def add(self, package):
        '''Add a packaged image to repository'''
        # check local repository
        if istools.pathtype(self.config.path) != "file":
            raise NotImplementedError("Repository addition must be local")
        # copy file to directory
        arrow("MD5summing tarballs", 1, self.verbose)
        # build dict of file to add
        filelist = dict()
        # script tarball
        arrow(package.filename, 2, self.verbose)
        filelist[package.md5] = package.path
        # data tarballs
        datas = package.datas
        for dt in datas:
            dt_path = datas[dt]["path"]
            old_md5 = datas[dt]["md5"]
            arrow(os.path.relpath(dt_path), 2, self.verbose)
            md5 = istools.md5sum(dt_path)
            if md5 != old_md5:
                raise Exception("MD5 mismatch on %s" % dt_path)
            filelist[md5] = dt_path
        # adding file to repository
        arrow("Adding files to directory", 1, self.verbose)
        for md5 in filelist:
            dest = os.path.join(self.config.path, md5)
            source = filelist[md5]
            if os.path.exists(dest):
                arrow("Skipping %s: already exists" % (os.path.basename(source)),
                      2, self.verbose)
            else:
                arrow("Adding %s (%s)" % (os.path.basename(source), md5), 2, self.verbose)
                istools.copy(source, dest,
                             self.config.chown, self.config.chgroup, self.config.fchmod)
        # add description to db
        self.db.add(package)
        # update last file
        self.update_last()

    def delete(self, name, version):
        '''Delete an image from repository'''
        raise NotImplementedError()
        # check local repository
        if istools.pathtype(self.config.path) != "file":
            raise NotImplementedError("Repository deletion must be local")
        desc = self.db.find(name, version)
        if desc is None:
            error("Unable to find %s version %s in database" % (name, version))
        # removing script tarballs
        arrow("Removing script tarball", 1, self.verbose)
        tpath = os.path.join(self.config.path,
                             "%s-%s%s" % (name, version, Image.extension))
        if os.path.exists(tpath):
            os.unlink(tpath)
            arrow("%s removed" % os.path.basename(tpath), 2, self.verbose)
        # removing data tarballs
        arrow("Removing data tarballs", 1, self.verbose)
        for tb in self.db.databalls(name, version):
            tpath = os.path.join(self.config.data, tb)
            if os.path.exists(tpath):
                os.unlink(tpath)
                arrow("%s removed" % tb, 2, self.verbose)
        # removing metadata
        self.db.delete(name, version)
        # update last file
        arrow("Updating last file", 1, self.verbose)
        self.update_last()

    def get(self, name, version):
        '''return a package from a name and version of pakage'''
        desc = self.db.get(name, version)
        p = PackageImage(os.path.join(self.config.path, desc["md5"]), verbose=self.verbose)
        if p.md5 != desc["md5"]:
            raise Exception("Invalid package MD5")
        return p

class RepositoryConfig(object):
    '''Repository configuration container'''

    def __init__(self, *args, **kwargs):
        # set default value for arguments
        self.name = args[0]
        self.dbname = "db"
        self.lastname = "last"
        self.path = ""
        self.chown = os.getuid()
        self.chgroup = os.getgid()
        umask = os.umask(0)
        os.umask(umask)
        self.fchmod =  0666 & ~umask
        self.dchmod =  0777 & ~umask
        self.update(**kwargs)

    def update(self, *args, **kwargs):
        '''Update attribute with checking value
        All attribute must already exists
        '''
        # autoset parameter in cmdline
        for k in kwargs:
            if hasattr(self, k):
                # attribute which are not in the following list cannot be loaded
                # from configuration
                try:
                    # convert userid
                    if k == "chown":
                        if not k.isdigit():
                            kwargs[k] = pwd.getpwnam(kwargs[k]).pw_uid
                        setattr(self, k, int(kwargs[k]))
                    # convert gid
                    elif k == "chgroup":
                        if not k.isdigit():
                            kwargs[k] = grp.getgrnam(kwargs[k]).gr_gid
                        setattr(self, k, int(kwargs[k]))
                    # convert file mode
                    elif k in ("fchmod", "dchmod"):
                        setattr(self, k, int(kwargs[k], 8))
                        # convert repo path
                    elif k in ("image", "data"):
                        setattr(self, k, istools.abspath(kwargs[k]))
                    # else is string
                    else:
                        setattr(self, k, kwargs[k])
                except Exception as e:
                    warn("Unable to set config parameter %s in repository %s: %s" % (k, self.name, e))

    def __eq__(self, other):
        return vars(self) == vars(other)

    def __ne__(self, other):
        return not (self == other)

    def __contains__(self, key):
        return key in self.__dict__

class RepositoryManager(object):
    '''Manage multiple repostories'''

    def __init__(self, timeout=None, verbose=True):
        self.verbose = verbose
        self.timeout = 3 if timeout is None else timeout
        self.repos = {}

    def register(self, configs):
        '''Register a list of repository from its config'''
        for conf in configs:
            self.repos[conf.name] = Repository(conf, self.verbose)

    def find_image(self, name, version):
        '''Find a repository containing image'''
        if version is None:
            arrow("Searching last version of %s" % name, 1, self.verbose)
        else:
            arrow("Searching %s version %s " % (name, version), 1, self.verbose)
        img = None
        # search in all repositories
        desc = None
        for repo in self.repos:
            desc = self.repos[repo].db.find(name, version)
            if desc is not None:
                # \o/
                break
        if desc is None:
            arrow("Not found", 2, self.verbose)
            if version is None:
                error("Unable to find a version of image %s" % name)
            else:
                error("Unable to find image %s version %s" % (name, version))
        arrow("Found %s version %s " % (desc["name"], desc["version"]), 2, self.verbose)
        return (desc, self.repos[repo])

    def get(self, name, version=None):
        '''Return a package object from local cache'''
        # find an image name/version in repository
        (desc, repo) = self.find_image(name, version)
        # get pkg object
        return repo.get(desc["name"], desc["version"])

class RepositoryCache(object):
    '''Local repository cache class'''

    def __init__(self, cache_path, timeout=3, verbose=True):
        self.verbose = verbose
        self.timeout = timeout
        self.repos = {}
        self.path = os.path.abspath(cache_path)
        # ensure cache directories are avaiblable
        if not os.path.exists(self.path):
            os.mkdir(self.path)
        if not os.access(self.path, os.W_OK | os.X_OK):
            raise Exception("%s is not writable or executable" % path)
        debug("Repository cache is in %s" % self.path)

    def register(self, configs):
        '''Register a list of repository from its config'''
        for conf in configs:
            self.repos[conf.name] = {}
            # keep original repository conf
            self.repos[conf.name]["orig"] = Repository(conf, self.verbose)
            # change configuration to make remote repository in cache
            cconf = copy.copy(conf)
            cconf.image = os.path.join(self.path, conf.name)
            cconf.data = "/dev/null"
            self.repos[conf.name]["cache"] = Repository(cconf, self.verbose)
            # create a local directory
            if not os.path.exists(cconf.image):
                os.mkdir(cconf.image)

    def update(self):
        '''Update cache info'''
        arrow("Updating repositories", 1, self.verbose)
        for r in self.repos:
            # last local
            local_last = self.last(r)
            # copy last file
            arrow("Checking %s repository last" % r, 2, self.verbose)
            istools.copy(self.repos[r]["orig"].last_path,
                         self.repos[r]["cache"].last_path, timeout=self.timeout)
            # last after update
            remote_last = self.last(r)
            debug("%s: last: local: %s, remote:%s" % (r, local_last, remote_last))
            # Updating db?
            remote_db = self.repos[r]["orig"].db.path
            local_db = self.repos[r]["cache"].db.path
            if remote_last > local_last or not os.path.exists(local_db):
                # copy db file
                arrow("Copying %s repository db" % r, 2, self.verbose)
                istools.copy(remote_db, local_db, timeout=self.timeout)

    def last(self, reponame):
        '''Return the last timestamp of a repo'''
        last_path = os.path.join(self.path, reponame, "last")
        try:
            return int(open(last_path, "r").read().rstrip())
        except Exception:
            return 0

    def get_image(self, reponame, imagename, imageversion):
        '''Obtain a local path in cache for a remote image in repo'''
        arrow("Getting image", 1, self.verbose)
        filename = "%s-%s%s" % (imagename, imageversion, Image.image_extension)
        cachepath = os.path.join(self.repos[reponame]["cache"].config.image, filename)
        # return db path if exists
        if os.path.exists(cachepath):
            arrow("Found in cache", 2, self.verbose)
        else:
            # get remote image
            remotepath = os.path.join(self.repos[reponame]["orig"].config.image, filename)
            arrow("Copying from repository", 2, self.verbose)
            istools.copy(remotepath, cachepath, timeout=self.timeout)
        return cachepath

    def find_image(self, name, version):
        '''Find an image in repositories'''
        if version is None:
            arrow("Searching last version of %s" % name, 1, self.verbose)
        else:
            arrow("Searching %s version %s " % (name, version), 1, self.verbose)
        img = None
        # search in all repositories
        for repo in self.repos:
            tempdb = Database(self.repos[repo]["cache"].db.path, False)
            img = tempdb.find(name, version)
            if img is not None:
                # \o/
                break
        if img is None:
            arrow("Not found", 2, self.verbose)
            if version is None:
                error("Unable to find a version of image %s" % name)
            else:
                error("Unable to find image %s version %s" % (name, version))
        arrow("Found %s version %s " % (img[0], img[1]), 2, self.verbose)
        return (repo, img[0], img[1])

    def get(self, name, version=None):
        '''Return a package object from local cache'''
        r, n, v = self.find_image(name, version)
        # download image if not in cache
        path = self.get_image(r, n, v)
        # create an object image
        return PackageImage(path, self.verbose)