From 56a216b0b275668c9174a1703122b0f0450e8cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Dunand?= Date: Fri, 12 Apr 2013 17:55:20 +0200 Subject: [PATCH] Allow to compress payload with different compressor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compressors are specified in description file. For example: [compressor] gzip = * none = *.gz, *.bz2 bzip2 = rootfs* This description file set gzip as default compressor, do not compress payload which end with .gz and .bz2 and use bzip2 as compressor for payload matching 'rootfs*' Signed-off-by: Sébastien Luttringer --- debian/control | 3 +- doc/is.1.rst | 12 ++++++- installsystems/image.py | 64 +++++++++++++++++++++++++++++--------- installsystems/template.py | 3 ++ installsystems/tools.py | 6 ++-- 5 files changed, 70 insertions(+), 18 deletions(-) diff --git a/debian/control b/debian/control index 2faeeb6..6604ad7 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,8 @@ Description: Python2 Installation framework Package: python-installsystems Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj +Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj, bzip2, xz-utils +Recommends: pbzip2 XB-Python-Version: ${python:Versions} Description: Python2 Installation framework - Python2 modules This package provides InstallSystems Python modules diff --git a/doc/is.1.rst b/doc/is.1.rst index f35308e..df4468f 100644 --- a/doc/is.1.rst +++ b/doc/is.1.rst @@ -386,9 +386,19 @@ InstallSystems use two kind of images: | author = Toto | is_min_version = 9 + Description file can also specify the compressor to use for payloads. Four compressors are available: 'none' (no compression), 'gzip', 'bzip2' and 'xz'. For each compressor, you can declare a globbing pattern to select specific payloads (use commas to separate patterns). Be careful, order matters. Here is an example: + + | + | [compressor] + | gzip = \* + | xz = rootfs\* + | none = \*.gz, \*.bz2, \*.xz + + The default compressor will be gzip, xz will be used for payload matching rootfs\* and each payload whose name ends with .gz, .bz2 and .xz will not be compressed. + **packaged image** - Built images are called packaged images. They are versionned, gzipped and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs: + Built images are called packaged images. They are versionned, compressed and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs: image_name.isimage This tarball contains build/, parser/, setup/, description and changelog. diff --git a/installsystems/image.py b/installsystems/image.py index 2a6d4c4..d7f3ddb 100644 --- a/installsystems/image.py +++ b/installsystems/image.py @@ -26,6 +26,7 @@ import configobj import cStringIO import difflib import imp +import fnmatch import json import locale import math @@ -56,6 +57,9 @@ version = IS_version description = string author = string is_min_version = IS_min_version + +[compressor] +__many__ = force_list """ @@ -68,6 +72,7 @@ class Image(object): # before version 6, it's strict string comparaison format = "1" extension = ".isimage" + default_compressor = "gzip" @staticmethod def check_image_name(buf): @@ -266,7 +271,8 @@ class SourceImage(Image): "version": "1", "description": "", "author": "", - "is_min_version": installsystems.version}} + "is_min_version": installsystems.version, + "compressor": "gzip = *\nnone = *.gz, *.bz2, *.xz"}} # create changelog example from template examples["changelog"] = {"path": "changelog", "content": istemplate.changelog} # create build example from template @@ -427,6 +433,7 @@ class SourceImage(Image): ans["gid"] = source_stat.st_gid ans["mode"] = stat.S_IMODE(source_stat.st_mode) ans["mtime"] = source_stat.st_mtime + ans["compressor"] = self.compressor(name) return ans def select_payloads(self): @@ -466,10 +473,12 @@ class SourceImage(Image): if not os.path.exists(paydesc["dest_path"]): if paydesc["isdir"]: self.create_payload_tarball(paydesc["dest_path"], - paydesc["source_path"]) + paydesc["source_path"], + paydesc["compressor"]) else: self.create_payload_file(paydesc["dest_path"], - paydesc["source_path"]) + paydesc["source_path"], + paydesc["compressor"]) # create versionned payload file if os.path.lexists(paydesc["link_path"]): os.unlink(paydesc["link_path"]) @@ -477,13 +486,13 @@ class SourceImage(Image): except Exception as e: raise ISError(u"Unable to create payload %s" % payload_name, e) - def create_payload_tarball(self, tar_path, data_path): + def create_payload_tarball(self, tar_path, data_path, compressor): ''' Create a payload tarball ''' try: # get compressor argv (first to escape file creation if not found) - a_comp = istools.get_compressor_path(self.compressor, compress=True) + a_comp = istools.get_compressor_path(compressor, compress=True) a_tar = ["tar", "--create", "--numeric-owner", "--directory", data_path, "."] # create destination file @@ -511,13 +520,13 @@ class SourceImage(Image): os.unlink(tar_path) raise - def create_payload_file(self, dest, source): + def create_payload_file(self, dest, source, compressor): ''' Create a payload file ''' try: # get compressor argv (first to escape file creation if not found) - a_comp = istools.get_compressor_path(self.compressor, compress=True) + a_comp = istools.get_compressor_path(compressor, compress=True) # open source file f_src = open(source, "r") # create destination file @@ -633,6 +642,8 @@ class SourceImage(Image): arrowlevel(1) # copy description desc = self.description.copy() + # only store compressor patterns + desc["compressor"] = desc["compressor"]["patterns"] # timestamp image arrow("Timestamping") desc["date"] = int(time.time()) @@ -657,7 +668,8 @@ class SourceImage(Image): "uid": payload_desc["uid"], "gid": payload_desc["gid"], "mode": payload_desc["mode"], - "mtime": payload_desc["mtime"] + "mtime": payload_desc["mtime"], + "compressor": payload_desc["compressor"] } arrowlevel(-1) # check md5 are uniq @@ -692,6 +704,17 @@ class SourceImage(Image): installsystems.printer.error('Wrong description file, %s %s: %s' % (section, optname, error)) for n in ("name","version", "description", "author", "is_min_version"): d[n] = cp["image"][n] + d["compressor"] = {} + # set payload compressor + d["compressor"]["patterns"] = cp["compressor"].items() + if not d["compressor"]["patterns"]: + d["compressor"]["patterns"] = [(Image.default_compressor, "*")] + for compressor, patterns in cp["compressor"].items(): + # is a valid compressor? + istools.get_compressor_path(compressor) + for pattern in patterns: + for payname in fnmatch.filter(self.select_payloads(), pattern): + d["compressor"][payname] = compressor except Exception as e: raise ISError(u"Bad description", e) return d @@ -714,13 +737,15 @@ class SourceImage(Image): raise ISError(u"Bad changelog", e) return cl - @property - def compressor(self): + def compressor(self, payname): ''' - Return image compressor + Return payload compressor ''' - # currently only support gzip - return "gzip" + try: + return self.description["compressor"][payname] + except KeyError: + # set default compressor if no compressor is specified + return Image.default_compressor class PackageImage(Image): @@ -846,6 +871,17 @@ class PackageImage(Image): desc.update(json.loads(img_desc)) self.check_image_name(desc["name"]) self.check_image_version(desc["version"]) + if "compressor" not in desc: + desc["compressor"] = "gzip = *" + else: + # format compressor pattern string + compressor_str = "" + for compressor, patterns in desc["compressor"]: + # if pattern is not empty + if patterns != ['']: + compressor_str += "%s = %s\n" % (compressor, ", ".join(patterns)) + # remove extra endline + desc["compressor"] = compressor_str[:-1] # add is_min_version if not present if "is_min_version" not in desc: desc["is_min_version"] = 0 @@ -1155,7 +1191,7 @@ class Payload(object): ''' Return payload compress format ''' - return self._compressor if self._compressor is not None else "gzip" + return self._compressor if self._compressor is not None else Image.default_compressor @property def info(self): diff --git a/installsystems/template.py b/installsystems/template.py index 3643f06..9e7961b 100644 --- a/installsystems/template.py +++ b/installsystems/template.py @@ -22,6 +22,9 @@ version = %(version)s description = %(description)s author = %(author)s is_min_version = %(is_min_version)s + +[compressor] +%(compressor)s """ changelog = u"""[1] diff --git a/installsystems/tools.py b/installsystems/tools.py index 9d2e884..f2e592c 100644 --- a/installsystems/tools.py +++ b/installsystems/tools.py @@ -671,11 +671,13 @@ def get_compressor_path(name, compress=True, level=None): Return better compressor argv from its generic compressor name e.g: bzip2 can return pbzip2 if available or bzip2 if not ''' - compressors = {"gzip": [["gzip", "--no-name", "--stdout"]], + compressors = {"none": [["cat"]], + "gzip": [["gzip", "--no-name", "--stdout"]], "bzip2": [["pbzip2", "--stdout"], ["bzip2", "--compress", "--stdout"]], "xz": [["xz", "--compress", "--stdout"]]} - decompressors = {"gzip": [["gzip", "--decompress", "--stdout"]], + decompressors = {"none": [["cat"]], + "gzip": [["gzip", "--decompress", "--stdout"]], "bzip2": [["pbzip2","--decompress", "--stdout"], ["bzip2", "--decompress", "--stdout"]], "xz": [["xz", "--decompress", "--stdout"]]} -- GitLab