From ba2b6b8559b00ca9f7dbd1dbefe29f3ebb890f42 Mon Sep 17 00:00:00 2001
From: Seblu <sebastien.luttringer@smartjog.com>
Date: Mon, 6 Jun 2011 12:17:41 +0200
Subject: [PATCH] Compression is now gzip
We stop using bzip2, because implemetnation of python 2.7 bz2 is uncompatible
with pbzip2. So tarball created with pbzip2 cannot be uncompressed with python.
So we need to find a speedy and portable compression algorithm. Gzip is the
best candidate
---
installsystems/database.py | 14 ++++++------
installsystems/image.py | 46 ++++++--------------------------------
installsystems/tools.py | 2 +-
3 files changed, 15 insertions(+), 47 deletions(-)
diff --git a/installsystems/database.py b/installsystems/database.py
index 6a7667e..b5dd846 100644
--- a/installsystems/database.py
+++ b/installsystems/database.py
@@ -26,7 +26,7 @@ class Database(object):
if os.path.exists(dbpath):
raise Exception("db already exists")
try:
- tarball = Tarball.open(dbpath, mode="w:bz2", dereference=True)
+ tarball = Tarball.open(dbpath, mode="w:gz", dereference=True)
tarball.add_str("format", Database.db_format, tarfile.REGTYPE, 0444)
tarball.close()
except Exception as e:
@@ -52,8 +52,8 @@ class Database(object):
jdesc = json.dumps(desc)
try:
arrow("Adding to tarball", 2, self.verbose)
- db = Tarball.open(self.path, mode='r:bz2')
- newdb = Tarball.open(newdb_path, mode='w:bz2')
+ db = Tarball.open(self.path, mode='r:gz')
+ newdb = Tarball.open(newdb_path, mode='w:gz')
for ti in db.getmembers():
if ti.name != name:
newdb.addfile(ti, db.extractfile(ti))
@@ -72,8 +72,8 @@ class Database(object):
newdb_path = "%s.new" % self.path
fname = "%s-%s.json" % (name, version)
try:
- db = Tarball.open(self.path, mode='r:bz2')
- newdb = Tarball.open(newdb_path, mode='w:bz2')
+ db = Tarball.open(self.path, mode='r:gz')
+ newdb = Tarball.open(newdb_path, mode='w:gz')
for ti in db.getmembers():
if ti.name != fname:
newdb.addfile(ti, db.extractfile(ti))
@@ -88,7 +88,7 @@ class Database(object):
def databalls(self, name, version):
'''List data tarballs filenames'''
try:
- db = Tarball.open(self.path, mode='r:bz2')
+ db = Tarball.open(self.path, mode='r:gz')
jdesc = json.loads(db.get_str("%s-%s.json" % (name, version)))
db.close()
return jdesc["data"]
@@ -98,7 +98,7 @@ class Database(object):
def find(self, name, version=None):
'''Find last version of an image'''
try:
- tarb = Tarball.open(self.path, mode='r:bz2')
+ tarb = Tarball.open(self.path, mode='r:gz')
candidates = [ int((os.path.splitext(tpname)[0]).rsplit("-", 1)[1])
for tpname in tarb.getnames("%s-\d+.json" % name) ]
tarb.close()
diff --git a/installsystems/image.py b/installsystems/image.py
index ac7e128..b645f68 100644
--- a/installsystems/image.py
+++ b/installsystems/image.py
@@ -37,22 +37,11 @@ class Image(object):
'''Check if @name is a valid image version'''
return re.match("\d+", buf) is not None
- def __init__(self, pbzip2=True):
- self.pbzip2_path = self.path_search("pbzip2") if pbzip2 else None
-
- def path_search(self, name, path=None):
- '''Search in PATH for a binary'''
- path = path or os.environ["PATH"]
- for d in path.split(os.pathsep):
- if os.path.exists(os.path.join(d, name)):
- return os.path.join(os.path.abspath(d), name)
- return None
-
class SourceImage(Image):
'''Image source manipulation class'''
@classmethod
- def create(cls, path, verbose=True, pbzip2=True):
+ def create(cls, path, verbose=True):
'''Create an empty source image'''
parser_path = os.path.join(path, "parser")
setup_path = os.path.join(path, "setup")
@@ -90,10 +79,10 @@ class SourceImage(Image):
os.chmod(pf, 0777 & ~umask)
except Exception as e:
raise Exception("Unable to set rights on %s: %s" % (pf, e))
- return cls(path, verbose, pbzip2)
+ return cls(path, verbose)
- def __init__(self, path, verbose=True, pbzip2=True):
- Image.__init__(self, pbzip2)
+ def __init__(self, path, verbose=True):
+ Image.__init__(self)
self.base_path = path
self.parser_path = os.path.join(path, "parser")
self.setup_path = os.path.join(path, "setup")
@@ -122,11 +111,6 @@ class SourceImage(Image):
# check if free to create script tarball
if os.path.exists(tarpath) and overwrite == False:
raise Exception("Tarball already exists. Remove it before")
- # printing pbzip2 status
- if self.pbzip2_path:
- arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path, 1, self.verbose)
- else:
- arrow("Parallel bzip disabled", 1, self.verbose)
# Create data tarballs
data_d = self.create_data_tarballs()
# generate description.json
@@ -135,7 +119,7 @@ class SourceImage(Image):
arrow("Creating scripts tarball", 1, self.verbose)
arrow("Name %s" % os.path.relpath(tarpath), 2, self.verbose)
try:
- tarball = Tarball.open(tarpath, mode="w:bz2", dereference=True)
+ tarball = Tarball.open(tarpath, mode="w:gz", dereference=True)
except Exception as e:
raise Exception("Unable to create tarball %s: %s" % (tarpath, e))
# add .description.json
@@ -190,26 +174,10 @@ class SourceImage(Image):
ddref = False if os.path.isdir(data_path) else True
try:
# opening file
- if self.pbzip2_path:
- tb = open(tar_path, mode="w")
- p = subprocess.Popen(self.pbzip2_path, shell=False, close_fds=True,
- stdin=subprocess.PIPE, stdout=tb.fileno())
- tarball = Tarball.open(mode="w|", dereference=ddref, fileobj=p.stdin)
- else:
- tarball = Tarball.open(tar_path, "w:bz2", dereference=ddref)
+ tarball = Tarball.open(tar_path, "w:gz", dereference=ddref)
tarball.add(data_path, arcname=dname, recursive=True)
# closing tarball file
tarball.close()
- if self.pbzip2_path:
- # closing pipe, needed to end pbzip2
- p.stdin.close()
- # waiting pbzip to terminate
- r = p.wait()
- # cleaning openfile
- tb.close()
- # status
- if r != 0:
- raise Exception("Data tarball %s creation return %s" % (tar_path, r))
except Exception as e:
raise Exception("Unable to create data tarball %s: %s" % (tar_path, e))
@@ -264,7 +232,7 @@ class PackageImage(Image):
self.path = os.path.abspath(path)
self.base_path = os.path.dirname(self.path)
self.verbose = verbose
- self.tarball = Tarball.open(self.path, mode='r:bz2')
+ self.tarball = Tarball.open(self.path, mode='r:gz')
self.parse()
@property
diff --git a/installsystems/tools.py b/installsystems/tools.py
index 28a1729..56c0867 100644
--- a/installsystems/tools.py
+++ b/installsystems/tools.py
@@ -101,7 +101,7 @@ def extractdata(image, name, target, filelist=None):
raise Exception("No such data tarball in %s" % image.name)
datainfo = image.datas[filename]
fileobject = ropen(filename)
- tarball = Tarball.open(fileobj=fileobject, mode="r|bz2")
+ tarball = Tarball.open(fileobj=fileobject, mode="r|gz")
if filelist is None:
tarball.extractall(target)
else:
--
GitLab