Commit 56a216b0 authored by Aurélien Dunand's avatar Aurélien Dunand Committed by Seblu

Allow to compress payload with different compressor

Compressors are specified in description file.
For example:

[compressor]
gzip = *
none = *.gz, *.bz2
bzip2 = rootfs*

This description file set gzip as default compressor, do not compress payload
which end with .gz and .bz2 and use bzip2 as compressor for payload matching
'rootfs*'
Signed-off-by: Seblu's avatarSébastien Luttringer <sebastien.luttringer@smartjog.com>
parent 8480fee9
...@@ -15,7 +15,8 @@ Description: Python2 Installation framework ...@@ -15,7 +15,8 @@ Description: Python2 Installation framework
Package: python-installsystems Package: python-installsystems
Architecture: all Architecture: all
Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj, bzip2, xz-utils
Recommends: pbzip2
XB-Python-Version: ${python:Versions} XB-Python-Version: ${python:Versions}
Description: Python2 Installation framework - Python2 modules Description: Python2 Installation framework - Python2 modules
This package provides InstallSystems Python modules This package provides InstallSystems Python modules
...@@ -386,9 +386,19 @@ InstallSystems use two kind of images: ...@@ -386,9 +386,19 @@ InstallSystems use two kind of images:
| author = Toto <toto@example.com> | author = Toto <toto@example.com>
| is_min_version = 9 | is_min_version = 9
Description file can also specify the compressor to use for payloads. Four compressors are available: 'none' (no compression), 'gzip', 'bzip2' and 'xz'. For each compressor, you can declare a globbing pattern to select specific payloads (use commas to separate patterns). Be careful, order matters. Here is an example:
|
| [compressor]
| gzip = \*
| xz = rootfs\*
| none = \*.gz, \*.bz2, \*.xz
The default compressor will be gzip, xz will be used for payload matching rootfs\* and each payload whose name ends with .gz, .bz2 and .xz will not be compressed.
**packaged image** **packaged image**
Built images are called packaged images. They are versionned, gzipped and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs: Built images are called packaged images. They are versionned, compressed and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs:
image_name.isimage image_name.isimage
This tarball contains build/, parser/, setup/, description and changelog. This tarball contains build/, parser/, setup/, description and changelog.
......
...@@ -26,6 +26,7 @@ import configobj ...@@ -26,6 +26,7 @@ import configobj
import cStringIO import cStringIO
import difflib import difflib
import imp import imp
import fnmatch
import json import json
import locale import locale
import math import math
...@@ -56,6 +57,9 @@ version = IS_version ...@@ -56,6 +57,9 @@ version = IS_version
description = string description = string
author = string author = string
is_min_version = IS_min_version is_min_version = IS_min_version
[compressor]
__many__ = force_list
""" """
...@@ -68,6 +72,7 @@ class Image(object): ...@@ -68,6 +72,7 @@ class Image(object):
# before version 6, it's strict string comparaison # before version 6, it's strict string comparaison
format = "1" format = "1"
extension = ".isimage" extension = ".isimage"
default_compressor = "gzip"
@staticmethod @staticmethod
def check_image_name(buf): def check_image_name(buf):
...@@ -266,7 +271,8 @@ class SourceImage(Image): ...@@ -266,7 +271,8 @@ class SourceImage(Image):
"version": "1", "version": "1",
"description": "", "description": "",
"author": "", "author": "",
"is_min_version": installsystems.version}} "is_min_version": installsystems.version,
"compressor": "gzip = *\nnone = *.gz, *.bz2, *.xz"}}
# create changelog example from template # create changelog example from template
examples["changelog"] = {"path": "changelog", "content": istemplate.changelog} examples["changelog"] = {"path": "changelog", "content": istemplate.changelog}
# create build example from template # create build example from template
...@@ -427,6 +433,7 @@ class SourceImage(Image): ...@@ -427,6 +433,7 @@ class SourceImage(Image):
ans["gid"] = source_stat.st_gid ans["gid"] = source_stat.st_gid
ans["mode"] = stat.S_IMODE(source_stat.st_mode) ans["mode"] = stat.S_IMODE(source_stat.st_mode)
ans["mtime"] = source_stat.st_mtime ans["mtime"] = source_stat.st_mtime
ans["compressor"] = self.compressor(name)
return ans return ans
def select_payloads(self): def select_payloads(self):
...@@ -466,10 +473,12 @@ class SourceImage(Image): ...@@ -466,10 +473,12 @@ class SourceImage(Image):
if not os.path.exists(paydesc["dest_path"]): if not os.path.exists(paydesc["dest_path"]):
if paydesc["isdir"]: if paydesc["isdir"]:
self.create_payload_tarball(paydesc["dest_path"], self.create_payload_tarball(paydesc["dest_path"],
paydesc["source_path"]) paydesc["source_path"],
paydesc["compressor"])
else: else:
self.create_payload_file(paydesc["dest_path"], self.create_payload_file(paydesc["dest_path"],
paydesc["source_path"]) paydesc["source_path"],
paydesc["compressor"])
# create versionned payload file # create versionned payload file
if os.path.lexists(paydesc["link_path"]): if os.path.lexists(paydesc["link_path"]):
os.unlink(paydesc["link_path"]) os.unlink(paydesc["link_path"])
...@@ -477,13 +486,13 @@ class SourceImage(Image): ...@@ -477,13 +486,13 @@ class SourceImage(Image):
except Exception as e: except Exception as e:
raise ISError(u"Unable to create payload %s" % payload_name, e) raise ISError(u"Unable to create payload %s" % payload_name, e)
def create_payload_tarball(self, tar_path, data_path): def create_payload_tarball(self, tar_path, data_path, compressor):
''' '''
Create a payload tarball Create a payload tarball
''' '''
try: try:
# get compressor argv (first to escape file creation if not found) # get compressor argv (first to escape file creation if not found)
a_comp = istools.get_compressor_path(self.compressor, compress=True) a_comp = istools.get_compressor_path(compressor, compress=True)
a_tar = ["tar", "--create", "--numeric-owner", "--directory", a_tar = ["tar", "--create", "--numeric-owner", "--directory",
data_path, "."] data_path, "."]
# create destination file # create destination file
...@@ -511,13 +520,13 @@ class SourceImage(Image): ...@@ -511,13 +520,13 @@ class SourceImage(Image):
os.unlink(tar_path) os.unlink(tar_path)
raise raise
def create_payload_file(self, dest, source): def create_payload_file(self, dest, source, compressor):
''' '''
Create a payload file Create a payload file
''' '''
try: try:
# get compressor argv (first to escape file creation if not found) # get compressor argv (first to escape file creation if not found)
a_comp = istools.get_compressor_path(self.compressor, compress=True) a_comp = istools.get_compressor_path(compressor, compress=True)
# open source file # open source file
f_src = open(source, "r") f_src = open(source, "r")
# create destination file # create destination file
...@@ -633,6 +642,8 @@ class SourceImage(Image): ...@@ -633,6 +642,8 @@ class SourceImage(Image):
arrowlevel(1) arrowlevel(1)
# copy description # copy description
desc = self.description.copy() desc = self.description.copy()
# only store compressor patterns
desc["compressor"] = desc["compressor"]["patterns"]
# timestamp image # timestamp image
arrow("Timestamping") arrow("Timestamping")
desc["date"] = int(time.time()) desc["date"] = int(time.time())
...@@ -657,7 +668,8 @@ class SourceImage(Image): ...@@ -657,7 +668,8 @@ class SourceImage(Image):
"uid": payload_desc["uid"], "uid": payload_desc["uid"],
"gid": payload_desc["gid"], "gid": payload_desc["gid"],
"mode": payload_desc["mode"], "mode": payload_desc["mode"],
"mtime": payload_desc["mtime"] "mtime": payload_desc["mtime"],
"compressor": payload_desc["compressor"]
} }
arrowlevel(-1) arrowlevel(-1)
# check md5 are uniq # check md5 are uniq
...@@ -692,6 +704,17 @@ class SourceImage(Image): ...@@ -692,6 +704,17 @@ class SourceImage(Image):
installsystems.printer.error('Wrong description file, %s %s: %s' % (section, optname, error)) installsystems.printer.error('Wrong description file, %s %s: %s' % (section, optname, error))
for n in ("name","version", "description", "author", "is_min_version"): for n in ("name","version", "description", "author", "is_min_version"):
d[n] = cp["image"][n] d[n] = cp["image"][n]
d["compressor"] = {}
# set payload compressor
d["compressor"]["patterns"] = cp["compressor"].items()
if not d["compressor"]["patterns"]:
d["compressor"]["patterns"] = [(Image.default_compressor, "*")]
for compressor, patterns in cp["compressor"].items():
# is a valid compressor?
istools.get_compressor_path(compressor)
for pattern in patterns:
for payname in fnmatch.filter(self.select_payloads(), pattern):
d["compressor"][payname] = compressor
except Exception as e: except Exception as e:
raise ISError(u"Bad description", e) raise ISError(u"Bad description", e)
return d return d
...@@ -714,13 +737,15 @@ class SourceImage(Image): ...@@ -714,13 +737,15 @@ class SourceImage(Image):
raise ISError(u"Bad changelog", e) raise ISError(u"Bad changelog", e)
return cl return cl
@property def compressor(self, payname):
def compressor(self):
''' '''
Return image compressor Return payload compressor
''' '''
# currently only support gzip try:
return "gzip" return self.description["compressor"][payname]
except KeyError:
# set default compressor if no compressor is specified
return Image.default_compressor
class PackageImage(Image): class PackageImage(Image):
...@@ -846,6 +871,17 @@ class PackageImage(Image): ...@@ -846,6 +871,17 @@ class PackageImage(Image):
desc.update(json.loads(img_desc)) desc.update(json.loads(img_desc))
self.check_image_name(desc["name"]) self.check_image_name(desc["name"])
self.check_image_version(desc["version"]) self.check_image_version(desc["version"])
if "compressor" not in desc:
desc["compressor"] = "gzip = *"
else:
# format compressor pattern string
compressor_str = ""
for compressor, patterns in desc["compressor"]:
# if pattern is not empty
if patterns != ['']:
compressor_str += "%s = %s\n" % (compressor, ", ".join(patterns))
# remove extra endline
desc["compressor"] = compressor_str[:-1]
# add is_min_version if not present # add is_min_version if not present
if "is_min_version" not in desc: if "is_min_version" not in desc:
desc["is_min_version"] = 0 desc["is_min_version"] = 0
...@@ -1155,7 +1191,7 @@ class Payload(object): ...@@ -1155,7 +1191,7 @@ class Payload(object):
''' '''
Return payload compress format Return payload compress format
''' '''
return self._compressor if self._compressor is not None else "gzip" return self._compressor if self._compressor is not None else Image.default_compressor
@property @property
def info(self): def info(self):
......
...@@ -22,6 +22,9 @@ version = %(version)s ...@@ -22,6 +22,9 @@ version = %(version)s
description = %(description)s description = %(description)s
author = %(author)s author = %(author)s
is_min_version = %(is_min_version)s is_min_version = %(is_min_version)s
[compressor]
%(compressor)s
""" """
changelog = u"""[1] changelog = u"""[1]
......
...@@ -671,11 +671,13 @@ def get_compressor_path(name, compress=True, level=None): ...@@ -671,11 +671,13 @@ def get_compressor_path(name, compress=True, level=None):
Return better compressor argv from its generic compressor name Return better compressor argv from its generic compressor name
e.g: bzip2 can return pbzip2 if available or bzip2 if not e.g: bzip2 can return pbzip2 if available or bzip2 if not
''' '''
compressors = {"gzip": [["gzip", "--no-name", "--stdout"]], compressors = {"none": [["cat"]],
"gzip": [["gzip", "--no-name", "--stdout"]],
"bzip2": [["pbzip2", "--stdout"], "bzip2": [["pbzip2", "--stdout"],
["bzip2", "--compress", "--stdout"]], ["bzip2", "--compress", "--stdout"]],
"xz": [["xz", "--compress", "--stdout"]]} "xz": [["xz", "--compress", "--stdout"]]}
decompressors = {"gzip": [["gzip", "--decompress", "--stdout"]], decompressors = {"none": [["cat"]],
"gzip": [["gzip", "--decompress", "--stdout"]],
"bzip2": [["pbzip2","--decompress", "--stdout"], "bzip2": [["pbzip2","--decompress", "--stdout"],
["bzip2", "--decompress", "--stdout"]], ["bzip2", "--decompress", "--stdout"]],
"xz": [["xz", "--decompress", "--stdout"]]} "xz": [["xz", "--decompress", "--stdout"]]}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment