Commit 56a216b0 authored by Aurélien Dunand's avatar Aurélien Dunand Committed by Sébastien Luttringer
Browse files

Allow to compress payload with different compressor 



Compressors are specified in description file.
For example:

[compressor]
gzip = *
none = *.gz, *.bz2
bzip2 = rootfs*

This description file set gzip as default compressor, do not compress payload
which end with .gz and .bz2 and use bzip2 as compressor for payload matching
'rootfs*'

Signed-off-by: default avatarSébastien Luttringer <sebastien.luttringer@smartjog.com>
parent 8480fee9

debian/control

+2 −1
Original line number Diff line number Diff line
@@ -15,7 +15,8 @@ Description: Python2 Installation framework 


Package: python-installsystems

Architecture: all

Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj

Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj, bzip2, xz-utils

Recommends: pbzip2

XB-Python-Version: ${python:Versions}

Description: Python2 Installation framework - Python2 modules

 This package provides InstallSystems Python modules

doc/is.1.rst

+11 −1
Original line number Diff line number Diff line
@@ -386,9 +386,19 @@ InstallSystems use two kind of images: 
       | author = Toto <toto@example.com>

       | is_min_version = 9



    Description file can also specify the compressor to use for payloads. Four compressors are available: 'none' (no compression), 'gzip', 'bzip2' and 'xz'. For each compressor, you can declare a globbing pattern to select specific payloads (use commas to separate patterns). Be careful, order matters. Here is an example:



        |

        | [compressor]

        | gzip = \*

        | xz = rootfs\*

        | none = \*.gz, \*.bz2, \*.xz



    The default compressor will be gzip, xz will be used for payload matching rootfs\* and each payload whose name ends with .gz, .bz2 and .xz will not be compressed.



**packaged image**



    Built images are called packaged images. They are versionned, gzipped and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs:

    Built images are called packaged images. They are versionned, compressed and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs:



    image_name.isimage

        This tarball contains build/, parser/, setup/, description and changelog.

installsystems/image.py

+50 −14
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ import configobj 
import cStringIO

import difflib

import imp

import fnmatch

import json

import locale

import math
@@ -56,6 +57,9 @@ version = IS_version 
description = string

author = string

is_min_version = IS_min_version



[compressor]

__many__ = force_list

"""
@@ -68,6 +72,7 @@ class Image(object): 
    # before version 6, it's strict string comparaison

    format = "1"

    extension = ".isimage"

    default_compressor = "gzip"



    @staticmethod

    def check_image_name(buf):
@@ -266,7 +271,8 @@ class SourceImage(Image): 
                "version": "1",

                "description": "",

                "author": "",

                "is_min_version": installsystems.version}}

                "is_min_version": installsystems.version,

                "compressor": "gzip = *\nnone = *.gz, *.bz2, *.xz"}}

        # create changelog example from template

        examples["changelog"] = {"path": "changelog", "content": istemplate.changelog}

        # create build example from template
@@ -427,6 +433,7 @@ class SourceImage(Image): 
        ans["gid"] = source_stat.st_gid

        ans["mode"] = stat.S_IMODE(source_stat.st_mode)

        ans["mtime"] = source_stat.st_mtime

        ans["compressor"] = self.compressor(name)

        return ans



    def select_payloads(self):
@@ -466,10 +473,12 @@ class SourceImage(Image): 
                if not os.path.exists(paydesc["dest_path"]):

                    if paydesc["isdir"]:

                        self.create_payload_tarball(paydesc["dest_path"],

                                                    paydesc["source_path"])

                                                    paydesc["source_path"],

                                                    paydesc["compressor"])

                    else:

                        self.create_payload_file(paydesc["dest_path"],

                                                 paydesc["source_path"])

                                                 paydesc["source_path"],

                                                 paydesc["compressor"])

                # create versionned payload file

                if os.path.lexists(paydesc["link_path"]):

                    os.unlink(paydesc["link_path"])
@@ -477,13 +486,13 @@ class SourceImage(Image): 
            except Exception as e:

                raise ISError(u"Unable to create payload %s" % payload_name, e)



    def create_payload_tarball(self, tar_path, data_path):

    def create_payload_tarball(self, tar_path, data_path, compressor):

        '''

        Create a payload tarball

        '''

        try:

            # get compressor argv (first to escape file creation if not found)

            a_comp = istools.get_compressor_path(self.compressor, compress=True)

            a_comp = istools.get_compressor_path(compressor, compress=True)

            a_tar = ["tar", "--create", "--numeric-owner", "--directory",

                     data_path, "."]

            # create destination file
@@ -511,13 +520,13 @@ class SourceImage(Image): 
                os.unlink(tar_path)

            raise



    def create_payload_file(self, dest, source):

    def create_payload_file(self, dest, source, compressor):

        '''

        Create a payload file

        '''

        try:

            # get compressor argv (first to escape file creation if not found)

            a_comp = istools.get_compressor_path(self.compressor, compress=True)

            a_comp = istools.get_compressor_path(compressor, compress=True)

            # open source file

            f_src = open(source, "r")

            # create destination file
@@ -633,6 +642,8 @@ class SourceImage(Image): 
        arrowlevel(1)

        # copy description

        desc = self.description.copy()

        # only store compressor patterns

        desc["compressor"] = desc["compressor"]["patterns"]

        # timestamp image

        arrow("Timestamping")

        desc["date"] = int(time.time())
@@ -657,7 +668,8 @@ class SourceImage(Image): 
                "uid": payload_desc["uid"],

                "gid": payload_desc["gid"],

                "mode": payload_desc["mode"],

                "mtime": payload_desc["mtime"]

                "mtime": payload_desc["mtime"],

                "compressor": payload_desc["compressor"]

                }

        arrowlevel(-1)

        # check md5 are uniq
@@ -692,6 +704,17 @@ class SourceImage(Image): 
                        installsystems.printer.error('Wrong description file, %s %s: %s' % (section, optname, error))

            for n in ("name","version", "description", "author", "is_min_version"):

                d[n] = cp["image"][n]

            d["compressor"] = {}

            # set payload compressor

            d["compressor"]["patterns"] = cp["compressor"].items()

            if not d["compressor"]["patterns"]:

                d["compressor"]["patterns"] = [(Image.default_compressor, "*")]

            for compressor, patterns in cp["compressor"].items():

                # is a valid compressor?

                istools.get_compressor_path(compressor)

                for pattern in patterns:

                    for payname in fnmatch.filter(self.select_payloads(), pattern):

                        d["compressor"][payname] = compressor

        except Exception as e:

            raise ISError(u"Bad description", e)

        return d
@@ -714,13 +737,15 @@ class SourceImage(Image): 
            raise ISError(u"Bad changelog", e)

        return cl



    @property

    def compressor(self):

    def compressor(self, payname):

        '''

        Return image compressor

        Return payload compressor

        '''

        # currently only support gzip

        return "gzip"

        try:

            return self.description["compressor"][payname]

        except KeyError:

            # set default compressor if no compressor is specified

            return Image.default_compressor





class PackageImage(Image):
@@ -846,6 +871,17 @@ class PackageImage(Image): 
            desc.update(json.loads(img_desc))

            self.check_image_name(desc["name"])

            self.check_image_version(desc["version"])

            if "compressor" not in desc:

                desc["compressor"] = "gzip = *"

            else:

                # format compressor pattern string

                compressor_str = ""

                for compressor, patterns in desc["compressor"]:

                    # if pattern is not empty

                    if patterns != ['']:

                        compressor_str += "%s = %s\n" % (compressor, ", ".join(patterns))

                # remove extra endline

                desc["compressor"] = compressor_str[:-1]

            # add is_min_version if not present

            if "is_min_version" not in desc:

                desc["is_min_version"] = 0
@@ -1155,7 +1191,7 @@ class Payload(object): 
        '''

        Return payload compress format

        '''

        return self._compressor if self._compressor is not None else "gzip"

        return self._compressor if self._compressor is not None else Image.default_compressor



    @property

    def info(self):

installsystems/template.py

+3 −0
Original line number Diff line number Diff line
@@ -22,6 +22,9 @@ version = %(version)s 
description = %(description)s

author = %(author)s

is_min_version = %(is_min_version)s



[compressor]

%(compressor)s

"""



changelog = u"""[1]

installsystems/tools.py

+4 −2
Original line number Diff line number Diff line
@@ -671,11 +671,13 @@ def get_compressor_path(name, compress=True, level=None): 
    Return better compressor argv from its generic compressor name

    e.g: bzip2 can return pbzip2 if available or bzip2 if not

    '''

    compressors = {"gzip": [["gzip", "--no-name", "--stdout"]],

    compressors = {"none": [["cat"]],

                   "gzip": [["gzip", "--no-name", "--stdout"]],

                   "bzip2": [["pbzip2", "--stdout"],

                             ["bzip2", "--compress", "--stdout"]],

                   "xz": [["xz", "--compress", "--stdout"]]}

    decompressors = {"gzip": [["gzip", "--decompress", "--stdout"]],

    decompressors = {"none": [["cat"]],

                     "gzip": [["gzip", "--decompress", "--stdout"]],

                     "bzip2": [["pbzip2","--decompress", "--stdout"],

                               ["bzip2", "--decompress", "--stdout"]],

                     "xz": [["xz", "--decompress", "--stdout"]]}