Commit 5fd0ead5 authored by Seblu's avatar Seblu
Browse files

Introduce RepositoryManager and extractdata 

This commit change a lot of things in installsystems way of life.
Repository cache is now disabled on access to remote repository done directly.
New database and tarball format
parent fede8fb5

bin/isimage

+1 −2
Original line number Diff line number Diff line
@@ -9,11 +9,10 @@ InstallSystems Image Manipulation Tool 
import os

import time

import datetime

import argparse

import installsystems

from installsystems.printer import *

from installsystems.image import SourceImage

# The following import can be removed when min version will be python 2.7

import installsystems.argparse as argparse



class DebugAction(argparse.Action):

    '''Set installsystems in debug mode. Argparse callback'''

bin/isinstall

+7 −9
Original line number Diff line number Diff line
@@ -9,14 +9,14 @@ InstallSystems Installation Tool 
import os

import time

import datetime

import argparse

import installsystems

import installsystems.tools as istools

from installsystems.printer import *

from installsystems.repository import RepositoryCache

from installsystems.repository import RepositoryManager

from installsystems.image import PackageImage

from installsystems.config import ConfigFile

# The following import can be removed when min version will be python 2.7

import installsystems.argparse as argparse





class DebugAction(argparse.Action):

    '''Set installsystems in debug mode. Argparse callback'''
@@ -38,7 +38,7 @@ p_main.add_argument("-c", "--config", dest="config", type=str, default=None, 
                    help="config file path")

p_main.add_argument("-v", "--image-version", dest="image_version", type=int, default=None,

                    help="image version")

p_main.add_argument("-t", "--timeout", dest="timeout", type=int, default=3,

p_main.add_argument("-t", "--timeout", dest="timeout", type=int, default=None,

                    help="download timeout")

p_main.add_argument("image_name", type=str,

                    help="image to install (path or name)")
@@ -53,13 +53,11 @@ try: 
        pkg = PackageImage(istools.abspath(args.image_name))

    elif image_name_type == "name":

        # init repo cache object

        repocache = RepositoryCache(config.cache, timeout=args.timeout, verbose=args.verbose)

        repoman = RepositoryManager(timeout=args.timeout, verbose=args.verbose)

        # register repositories

        repocache.register(config.repos)

        # update remote info

        repocache.update()

        repoman.register(config.repos)

        # get image package

        pkg = repocache.get(args.image_name, args.image_version)

        pkg = repoman.get(args.image_name, args.image_version)

    else:

        p_main.print_usage()

        exit(1)

bin/isrepo

+14 −20
Original line number Diff line number Diff line
@@ -7,13 +7,12 @@ InstallSystems Repository Manipulation Tool 
'''



import os

import argparse

import installsystems

from installsystems.printer import *

from installsystems.repository import Repository

from installsystems.repository import Repository, RepositoryConfig

from installsystems.image import PackageImage

from installsystems.config import ConfigFile

# The following import can be removed when min version will be python 2.7

import installsystems.argparse as argparse



class DebugAction(argparse.Action):

    '''Set installsystems in debug mode. Argparse callback'''
@@ -27,24 +26,24 @@ def init(args): 
    '''Create an empty fresh repo tree'''

    # call init from library

    try:

        Repository.create(args.repo, args.verbose)

        Repository.create(args.repository, args.verbose)

    except Exception as e:

        raise Exception("init failed: %s" % e)



def add(args):

    '''Add a package to repository'''

    try:

        repo = Repository(args.repo, args.verbose)

        repo = Repository(args.repository, args.verbose)

        pkg = PackageImage(args.path, args.verbose)

        pkg.check_md5()

        repo.add(pkg)

    except Exception as e:

        raise

        raise Exception("add failed: %s" % e)



def delete(args):

    '''Remove a package from repository'''

    try:

        repo = Repository(args.repo, args.verbose)

        repo = Repository(args.repository, args.verbose)

        repo.delete(args.image_name, args.image_version)

    except Exception as e:

        raise Exception("del failed: %s" % e)
@@ -60,24 +59,20 @@ p_main.add_argument('-q', "--quiet", action="store_false", dest="verbose", defau 
p_main.add_argument("-c", "--config", dest="config", type=str, default=None,

                    help="config file path")

p_main.add_argument("-r", "--repo-name", dest="repo_name", type=str, default=None,

                    help="repository name")

                    help="repository name in config file")

subparsers = p_main.add_subparsers()

# Init command parser

p_init = subparsers.add_parser("init", help=init.__doc__.lower())

p_init.add_argument("repo_image", nargs="?", default=argparse.SUPPRESS,

                    help="Path of the new image directory")

p_init.add_argument("repo_data", nargs="?", default=argparse.SUPPRESS,

                    help="Path of the new data directory")

p_init.set_defaults(func=init)

# Add command parser

p_add =  subparsers.add_parser("add", help=add.__doc__.lower())

p_add.add_argument("path", type=str)

p_add.set_defaults(func=add)

# Del command parser

p_del =  subparsers.add_parser("del", help=delete.__doc__.lower())

p_del.add_argument("image_name", type=str)

p_del.add_argument("image_version", type=str)

p_del.set_defaults(func=delete)

#p_del =  subparsers.add_parser("del", help=delete.__doc__.lower())

#p_del.add_argument("image_name", type=str)

#p_del.add_argument("image_version", type=str)

#p_del.set_defaults(func=delete)

try:

    # Parse and run

    args = p_main.parse_args()
@@ -89,13 +84,12 @@ try: 
    else:

        repos = config.repos

    if len(repos) == 1:

        args.repo = repos[0]

        args.repository = repos[0]

    elif len(repos) > 1:

        raise Exception("Please select a repository with -r")

    else:

    if len(repos) == 0 or not isinstance(args.repository, RepositoryConfig):

        raise Exception("No image repository found")

    debug("Image repo: %s" % args.repo.image)

    debug("Data repo: %s" % args.repo.data)

    debug("Repository: %s" % args.repository.path)

    args.func(args)

except Exception as e:

    p_main.print_usage()

installsystems/config.py

+1 −1
Original line number Diff line number Diff line
@@ -46,7 +46,7 @@ class ConfigFile(object): 
                # each section is a repository

                for rep in cp.sections():

                    # check if its a repo section

                    if "image" not in cp.options(rep):

                    if "path" not in cp.options(rep):

                        continue

                    # get all options in repo

                    self._repos.append(RepositoryConfig(rep, **dict(cp.items(rep))))

installsystems/database.py

+63 −18
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ import json 
import os

import shutil

import tarfile

import cStringIO

import installsystems.tools as istools

from installsystems.tarball import Tarball

from installsystems.printer import *
@@ -22,9 +23,12 @@ class Database(object): 
    @classmethod

    def create(cls, path, verbose=True):

        arrow("Creating repository database", 1, verbose)

        # check locality

        if istools.pathtype(path) != "file":

            raise NotImplementedError("Database creation must be local")

        dbpath = istools.abspath(path)

        if os.path.exists(dbpath):

            raise Exception("db already exists")

            raise Exception("Database already exists. Remove it before")

        try:

            tarball = Tarball.open(dbpath, mode="w:gz", dereference=True)

            tarball.add_str("format", Database.db_format, tarfile.REGTYPE, 0444)
@@ -36,39 +40,67 @@ class Database(object): 
    def __init__(self, path, verbose=True):

        self.path = istools.abspath(path)

        self.verbose = verbose

        # load db in memory

        self.file = cStringIO.StringIO()

        shutil.copyfileobj(istools.uopen(self.path), self.file)



    def get(self, name, version):

        '''Return a description dict from a package name'''

        # parse tarball

        try:

            self.file.seek(0)

            tarball = Tarball.open(fileobj=self.file, mode="r:gz")

            rdata = tarball.get_str("%s-%s" % (name, version))

            tarball.close()

        except KeyError:

            raise Exception("No package %s version %s in metadata" % (name, version))

        except Exception as e:

            raise Exception("Unable to read db %s version %s: e" % (name, version, e))

        # convert loaded data into dict (json parser)

        try:

            return json.loads(rdata)

        except Exception as e:

            raise Exception("Invalid metadata in package %s version %s: e" % (name, version, e))



    def add(self, package):

        '''Add a packaged image to a db'''

        arrow("Adding metadata to db", 1, self.verbose)

        # check locality

        if istools.pathtype(self.path) != "file":

            raise NotImplementedError("Database addition must be local")

        # naming

        name = "%s.json" % package.name

        newdb_path = "%s.new" % self.path

        # compute md5

        arrow("Compute MD5 of %s" % os.path.relpath(package.path), 2, self.verbose)

        md5 = package.md5

        arrow("Formating metadata", 2, self.verbose)

        desc = package.description

        desc["md5"] = md5

        desc["md5"] = package.md5

        jdesc = json.dumps(desc)

        try:

            arrow("Adding to tarball", 2, self.verbose)

            db = Tarball.open(self.path, mode='r:gz')

            newdb = Tarball.open(newdb_path, mode='w:gz')

            arrow("Adding metadata", 2, self.verbose)

            self.file.seek(0)

            newfile = cStringIO.StringIO()

            db = Tarball.open(fileobj=self.file, mode='r:gz')

            newdb = Tarball.open(fileobj=newfile, mode='w:gz')

            for ti in db.getmembers():

                if ti.name != name:

                if ti.name != package.id:

                    newdb.addfile(ti, db.extractfile(ti))

            newdb.add_str(name, jdesc, tarfile.REGTYPE, 0444)

            newdb.add_str(package.id, jdesc, tarfile.REGTYPE, 0644)

            db.close()

            newdb.close()

            # preserve permission and stats when moving

            shutil.copystat(self.path, newdb_path)

            os.rename(newdb_path, self.path)

            # writing to disk

            arrow("Writing to disk", 2, self.verbose)

            self.file.close()

            self.file = newfile

            self.write()

        except Exception as e:

            raise Exception("Adding metadata fail: %s" % e)



    def delete(self, name, version):

        '''Deltete a packaged image'''

        arrow("Removing metadata from db", 1, self.verbose)

        # check locality

        if istools.pathtype(self.path) != "file":

            raise NotImplementedError("Database deletion must be local")

        newdb_path = "%s.new" % self.path

        fname = "%s-%s.json" % (name, version)

        try:
@@ -88,19 +120,21 @@ class Database(object): 
    def databalls(self, name, version):

        '''List data tarballs filenames'''

        try:

            db = Tarball.open(self.path, mode='r:gz')

            self.file.seek(0)

            db = Tarball.open(fileobj=self.file, mode='r:gz')

            jdesc = json.loads(db.get_str("%s-%s.json" % (name, version)))

            db.close()

            return jdesc["data"]

        except Exception as e:

            raise Exception("Listing data tarballs fail: %s" % e)

            raise Exception("List data tarballs fail: %s" % e)



    def find(self, name, version=None):

        '''Find last version of an image'''

        try:

            tarb = Tarball.open(self.path, mode='r:gz')

            self.file.seek(0)

            tarb = Tarball.open(fileobj=self.file, mode='r:gz')

            candidates = [ int((os.path.splitext(tpname)[0]).rsplit("-", 1)[1])

                           for tpname in tarb.getnames("%s-\d+.json" % name) ]

                           for tpname in tarb.getnames("%s-\d+" % name) ]

            tarb.close()

        except Exception as e:

            raise Exception("Find in db %s fail: %s" % (self.path, e))
@@ -113,4 +147,15 @@ class Database(object): 
        # check if version exists

        if int(version) not in candidates:

            return None

        return (name, version)
 
        return self.get(name, version)



    def write(self):

        '''Write current dabatase into its file'''

        if istools.pathtype(self.path) != "file":

            raise NotImplementedError("Database writing must be local")

        try:

            dest = open(self.path, "w")

            self.file.seek(0)

            shutil.copyfileobj(self.file, dest)

        except Exception as e:

            raise Exception("Unable to write database: %s" % e)