Commit 56a216b0 authored by Aurélien Dunand's avatar Aurélien Dunand Committed by Seblu

Allow to compress payload with different compressor

Compressors are specified in description file.
For example:

[compressor]
gzip = *
none = *.gz, *.bz2
bzip2 = rootfs*

This description file set gzip as default compressor, do not compress payload
which end with .gz and .bz2 and use bzip2 as compressor for payload matching
'rootfs*'
Signed-off-by: Seblu's avatarSébastien Luttringer <sebastien.luttringer@smartjog.com>
parent 8480fee9
......@@ -15,7 +15,8 @@ Description: Python2 Installation framework
Package: python-installsystems
Architecture: all
Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj
Depends: ${misc:Depends}, ${python:Depends}, python-paramiko, python-argparse (>= 1.2.1), python-progressbar (>= 2.3), python-jinja2, python-configobj, bzip2, xz-utils
Recommends: pbzip2
XB-Python-Version: ${python:Versions}
Description: Python2 Installation framework - Python2 modules
This package provides InstallSystems Python modules
......@@ -386,9 +386,19 @@ InstallSystems use two kind of images:
| author = Toto <toto@example.com>
| is_min_version = 9
Description file can also specify the compressor to use for payloads. Four compressors are available: 'none' (no compression), 'gzip', 'bzip2' and 'xz'. For each compressor, you can declare a globbing pattern to select specific payloads (use commas to separate patterns). Be careful, order matters. Here is an example:
|
| [compressor]
| gzip = \*
| xz = rootfs\*
| none = \*.gz, \*.bz2, \*.xz
The default compressor will be gzip, xz will be used for payload matching rootfs\* and each payload whose name ends with .gz, .bz2 and .xz will not be compressed.
**packaged image**
Built images are called packaged images. They are versionned, gzipped and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs:
Built images are called packaged images. They are versionned, compressed and ready to deploy. Like source images, package images still make the difference between scripts and payloads. But it doesn't make difference between build, parser and setup scripts. In fact you will have at least two tarballs:
image_name.isimage
This tarball contains build/, parser/, setup/, description and changelog.
......
......@@ -26,6 +26,7 @@ import configobj
import cStringIO
import difflib
import imp
import fnmatch
import json
import locale
import math
......@@ -56,6 +57,9 @@ version = IS_version
description = string
author = string
is_min_version = IS_min_version
[compressor]
__many__ = force_list
"""
......@@ -68,6 +72,7 @@ class Image(object):
# before version 6, it's strict string comparaison
format = "1"
extension = ".isimage"
default_compressor = "gzip"
@staticmethod
def check_image_name(buf):
......@@ -266,7 +271,8 @@ class SourceImage(Image):
"version": "1",
"description": "",
"author": "",
"is_min_version": installsystems.version}}
"is_min_version": installsystems.version,
"compressor": "gzip = *\nnone = *.gz, *.bz2, *.xz"}}
# create changelog example from template
examples["changelog"] = {"path": "changelog", "content": istemplate.changelog}
# create build example from template
......@@ -427,6 +433,7 @@ class SourceImage(Image):
ans["gid"] = source_stat.st_gid
ans["mode"] = stat.S_IMODE(source_stat.st_mode)
ans["mtime"] = source_stat.st_mtime
ans["compressor"] = self.compressor(name)
return ans
def select_payloads(self):
......@@ -466,10 +473,12 @@ class SourceImage(Image):
if not os.path.exists(paydesc["dest_path"]):
if paydesc["isdir"]:
self.create_payload_tarball(paydesc["dest_path"],
paydesc["source_path"])
paydesc["source_path"],
paydesc["compressor"])
else:
self.create_payload_file(paydesc["dest_path"],
paydesc["source_path"])
paydesc["source_path"],
paydesc["compressor"])
# create versionned payload file
if os.path.lexists(paydesc["link_path"]):
os.unlink(paydesc["link_path"])
......@@ -477,13 +486,13 @@ class SourceImage(Image):
except Exception as e:
raise ISError(u"Unable to create payload %s" % payload_name, e)
def create_payload_tarball(self, tar_path, data_path):
def create_payload_tarball(self, tar_path, data_path, compressor):
'''
Create a payload tarball
'''
try:
# get compressor argv (first to escape file creation if not found)
a_comp = istools.get_compressor_path(self.compressor, compress=True)
a_comp = istools.get_compressor_path(compressor, compress=True)
a_tar = ["tar", "--create", "--numeric-owner", "--directory",
data_path, "."]
# create destination file
......@@ -511,13 +520,13 @@ class SourceImage(Image):
os.unlink(tar_path)
raise
def create_payload_file(self, dest, source):
def create_payload_file(self, dest, source, compressor):
'''
Create a payload file
'''
try:
# get compressor argv (first to escape file creation if not found)
a_comp = istools.get_compressor_path(self.compressor, compress=True)
a_comp = istools.get_compressor_path(compressor, compress=True)
# open source file
f_src = open(source, "r")
# create destination file
......@@ -633,6 +642,8 @@ class SourceImage(Image):
arrowlevel(1)
# copy description
desc = self.description.copy()
# only store compressor patterns
desc["compressor"] = desc["compressor"]["patterns"]
# timestamp image
arrow("Timestamping")
desc["date"] = int(time.time())
......@@ -657,7 +668,8 @@ class SourceImage(Image):
"uid": payload_desc["uid"],
"gid": payload_desc["gid"],
"mode": payload_desc["mode"],
"mtime": payload_desc["mtime"]
"mtime": payload_desc["mtime"],
"compressor": payload_desc["compressor"]
}
arrowlevel(-1)
# check md5 are uniq
......@@ -692,6 +704,17 @@ class SourceImage(Image):
installsystems.printer.error('Wrong description file, %s %s: %s' % (section, optname, error))
for n in ("name","version", "description", "author", "is_min_version"):
d[n] = cp["image"][n]
d["compressor"] = {}
# set payload compressor
d["compressor"]["patterns"] = cp["compressor"].items()
if not d["compressor"]["patterns"]:
d["compressor"]["patterns"] = [(Image.default_compressor, "*")]
for compressor, patterns in cp["compressor"].items():
# is a valid compressor?
istools.get_compressor_path(compressor)
for pattern in patterns:
for payname in fnmatch.filter(self.select_payloads(), pattern):
d["compressor"][payname] = compressor
except Exception as e:
raise ISError(u"Bad description", e)
return d
......@@ -714,13 +737,15 @@ class SourceImage(Image):
raise ISError(u"Bad changelog", e)
return cl
@property
def compressor(self):
def compressor(self, payname):
'''
Return image compressor
Return payload compressor
'''
# currently only support gzip
return "gzip"
try:
return self.description["compressor"][payname]
except KeyError:
# set default compressor if no compressor is specified
return Image.default_compressor
class PackageImage(Image):
......@@ -846,6 +871,17 @@ class PackageImage(Image):
desc.update(json.loads(img_desc))
self.check_image_name(desc["name"])
self.check_image_version(desc["version"])
if "compressor" not in desc:
desc["compressor"] = "gzip = *"
else:
# format compressor pattern string
compressor_str = ""
for compressor, patterns in desc["compressor"]:
# if pattern is not empty
if patterns != ['']:
compressor_str += "%s = %s\n" % (compressor, ", ".join(patterns))
# remove extra endline
desc["compressor"] = compressor_str[:-1]
# add is_min_version if not present
if "is_min_version" not in desc:
desc["is_min_version"] = 0
......@@ -1155,7 +1191,7 @@ class Payload(object):
'''
Return payload compress format
'''
return self._compressor if self._compressor is not None else "gzip"
return self._compressor if self._compressor is not None else Image.default_compressor
@property
def info(self):
......
......@@ -22,6 +22,9 @@ version = %(version)s
description = %(description)s
author = %(author)s
is_min_version = %(is_min_version)s
[compressor]
%(compressor)s
"""
changelog = u"""[1]
......
......@@ -671,11 +671,13 @@ def get_compressor_path(name, compress=True, level=None):
Return better compressor argv from its generic compressor name
e.g: bzip2 can return pbzip2 if available or bzip2 if not
'''
compressors = {"gzip": [["gzip", "--no-name", "--stdout"]],
compressors = {"none": [["cat"]],
"gzip": [["gzip", "--no-name", "--stdout"]],
"bzip2": [["pbzip2", "--stdout"],
["bzip2", "--compress", "--stdout"]],
"xz": [["xz", "--compress", "--stdout"]]}
decompressors = {"gzip": [["gzip", "--decompress", "--stdout"]],
decompressors = {"none": [["cat"]],
"gzip": [["gzip", "--decompress", "--stdout"]],
"bzip2": [["pbzip2","--decompress", "--stdout"],
["bzip2", "--decompress", "--stdout"]],
"xz": [["xz", "--decompress", "--stdout"]]}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment