Commit ba2b6b85 authored by Seblu's avatar Seblu
Browse files

Compression is now gzip 

We stop using bzip2, because implemetnation of python 2.7 bz2 is uncompatible
with pbzip2. So tarball created with pbzip2 cannot be uncompressed with python.
So we need to find a speedy and portable compression algorithm. Gzip is the
best candidate
parent f5816a01

installsystems/database.py

+7 −7
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ class Database(object): 
        if os.path.exists(dbpath):

            raise Exception("db already exists")

        try:

            tarball = Tarball.open(dbpath, mode="w:bz2", dereference=True)

            tarball = Tarball.open(dbpath, mode="w:gz", dereference=True)

            tarball.add_str("format", Database.db_format, tarfile.REGTYPE, 0444)

            tarball.close()

        except Exception as e:
@@ -52,8 +52,8 @@ class Database(object): 
        jdesc = json.dumps(desc)

        try:

            arrow("Adding to tarball", 2, self.verbose)

            db = Tarball.open(self.path, mode='r:bz2')

            newdb = Tarball.open(newdb_path, mode='w:bz2')

            db = Tarball.open(self.path, mode='r:gz')

            newdb = Tarball.open(newdb_path, mode='w:gz')

            for ti in db.getmembers():

                if ti.name != name:

                    newdb.addfile(ti, db.extractfile(ti))
@@ -72,8 +72,8 @@ class Database(object): 
        newdb_path = "%s.new" % self.path

        fname = "%s-%s.json" % (name, version)

        try:

            db = Tarball.open(self.path, mode='r:bz2')

            newdb = Tarball.open(newdb_path, mode='w:bz2')

            db = Tarball.open(self.path, mode='r:gz')

            newdb = Tarball.open(newdb_path, mode='w:gz')

            for ti in db.getmembers():

                if ti.name != fname:

                    newdb.addfile(ti, db.extractfile(ti))
@@ -88,7 +88,7 @@ class Database(object): 
    def databalls(self, name, version):

        '''List data tarballs filenames'''

        try:

            db = Tarball.open(self.path, mode='r:bz2')

            db = Tarball.open(self.path, mode='r:gz')

            jdesc = json.loads(db.get_str("%s-%s.json" % (name, version)))

            db.close()

            return jdesc["data"]
@@ -98,7 +98,7 @@ class Database(object): 
    def find(self, name, version=None):

        '''Find last version of an image'''

        try:

            tarb = Tarball.open(self.path, mode='r:bz2')

            tarb = Tarball.open(self.path, mode='r:gz')

            candidates = [ int((os.path.splitext(tpname)[0]).rsplit("-", 1)[1])

                           for tpname in tarb.getnames("%s-\d+.json" % name) ]

            tarb.close()

installsystems/image.py

+7 −39
Original line number Diff line number Diff line
@@ -37,22 +37,11 @@ class Image(object): 
        '''Check if @name is a valid image version'''

        return re.match("\d+", buf) is not None



    def __init__(self, pbzip2=True):

        self.pbzip2_path = self.path_search("pbzip2") if pbzip2 else None



    def path_search(self, name, path=None):

        '''Search in PATH for a binary'''

        path = path or os.environ["PATH"]

        for d in path.split(os.pathsep):

            if os.path.exists(os.path.join(d, name)):

                return os.path.join(os.path.abspath(d), name)

        return None



class SourceImage(Image):

    '''Image source manipulation class'''



    @classmethod

    def create(cls, path, verbose=True, pbzip2=True):

    def create(cls, path, verbose=True):

        '''Create an empty source image'''

        parser_path = os.path.join(path, "parser")

        setup_path = os.path.join(path, "setup")
@@ -90,10 +79,10 @@ class SourceImage(Image): 
                    os.chmod(pf, 0777 & ~umask)

        except Exception as e:

            raise Exception("Unable to set rights on %s: %s" % (pf, e))

        return cls(path, verbose, pbzip2)

        return cls(path, verbose)



    def __init__(self, path, verbose=True, pbzip2=True):

        Image.__init__(self, pbzip2)

    def __init__(self, path, verbose=True):

        Image.__init__(self)

        self.base_path = path

        self.parser_path = os.path.join(path, "parser")

        self.setup_path = os.path.join(path, "setup")
@@ -122,11 +111,6 @@ class SourceImage(Image): 
        # check if free to create script tarball

        if os.path.exists(tarpath) and overwrite == False:

            raise Exception("Tarball already exists. Remove it before")

        # printing pbzip2 status

        if self.pbzip2_path:

            arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path, 1, self.verbose)

        else:

            arrow("Parallel bzip disabled", 1, self.verbose)

        #  Create data tarballs

        data_d = self.create_data_tarballs()

        # generate description.json
@@ -135,7 +119,7 @@ class SourceImage(Image): 
        arrow("Creating scripts tarball", 1, self.verbose)

        arrow("Name %s" % os.path.relpath(tarpath), 2, self.verbose)

        try:

            tarball = Tarball.open(tarpath, mode="w:bz2", dereference=True)

            tarball = Tarball.open(tarpath, mode="w:gz", dereference=True)

        except Exception as e:

            raise Exception("Unable to create tarball %s: %s" % (tarpath, e))

        # add .description.json
@@ -190,26 +174,10 @@ class SourceImage(Image): 
        ddref = False if os.path.isdir(data_path) else True

        try:

            # opening file

            if self.pbzip2_path:

                tb = open(tar_path, mode="w")

                p = subprocess.Popen(self.pbzip2_path, shell=False, close_fds=True,

                                     stdin=subprocess.PIPE, stdout=tb.fileno())

                tarball = Tarball.open(mode="w|", dereference=ddref, fileobj=p.stdin)

            else:

                tarball = Tarball.open(tar_path, "w:bz2", dereference=ddref)

            tarball = Tarball.open(tar_path, "w:gz", dereference=ddref)

            tarball.add(data_path, arcname=dname, recursive=True)

            # closing tarball file

            tarball.close()

            if self.pbzip2_path:

                # closing pipe, needed to end pbzip2

                p.stdin.close()

                # waiting pbzip to terminate

                r = p.wait()

                # cleaning openfile

                tb.close()

                # status

                if r != 0:

                    raise Exception("Data tarball %s creation return %s" % (tar_path, r))

        except Exception as e:

            raise Exception("Unable to create data tarball %s: %s" % (tar_path, e))
@@ -264,7 +232,7 @@ class PackageImage(Image): 
        self.path = os.path.abspath(path)

        self.base_path = os.path.dirname(self.path)

        self.verbose = verbose

        self.tarball = Tarball.open(self.path, mode='r:bz2')

        self.tarball = Tarball.open(self.path, mode='r:gz')

        self.parse()



    @property

installsystems/tools.py

+1 −1
Original line number Diff line number Diff line
@@ -101,7 +101,7 @@ def extractdata(image, name, target, filelist=None): 
        raise Exception("No such data tarball in %s" % image.name)

    datainfo = image.datas[filename]

    fileobject = ropen(filename)

    tarball = Tarball.open(fileobj=fileobject, mode="r|bz2")

    tarball = Tarball.open(fileobj=fileobject, mode="r|gz")

    if filelist is None:

        tarball.extractall(target)

    else: