diff --git a/installsystems/config.py b/installsystems/config.py
index 9b75f8fc2c806cded976d20adf2449942ba900ef..b9799112eaa1124de0aa94e79fc5a2ab0707824c 100644
--- a/installsystems/config.py
+++ b/installsystems/config.py
@@ -51,7 +51,6 @@ class ConfigFile(object):
# get all options in repo
self._repos.append(RepositoryConfig(rep, **dict(cp.items(rep))))
except Exception as e:
- raise
raise Exception("Unable load file %s: %s" % (self.path, e))
else:
debug("No config file found")
diff --git a/installsystems/database.py b/installsystems/database.py
index 874a27a46367108775a472a89d00ccfa9406d324..a774b144217ae491fc13aa426770cb43d7c0cb6a 100644
--- a/installsystems/database.py
+++ b/installsystems/database.py
@@ -52,7 +52,7 @@ class Database(object):
self.conn.execute("PRAGMA foreign_keys = ON")
def get(self, name, version):
- '''Return a description dict from a package name'''
+ '''Return a description dict from a image name'''
# parse tarball
try:
self.file.seek(0)
@@ -60,20 +60,20 @@ class Database(object):
rdata = tarball.get_str("%s-%s" % (name, version))
tarball.close()
except KeyError:
- raise Exception("No package %s version %s in metadata" % (name, version))
+ raise Exception("No image %s version %s in metadata" % (name, version))
except Exception as e:
raise Exception("Unable to read db %s version %s: %s" % (name, version, e))
# convert loaded data into dict (json parser)
try:
return json.loads(rdata)
except Exception as e:
- raise Exception("Invalid metadata in package %s version %s: e" % (name, version, e))
+ raise Exception("Invalid metadata in image %s version %s: e" % (name, version, e))
def ask(self, sql, args=()):
'''Ask question to db'''
return self.conn.execute(sql, args)
- def add(self, package):
+ def add(self, image):
'''Add a packaged image to a db'''
try:
# let's go
@@ -82,28 +82,28 @@ class Database(object):
# insert image information
arrow("Add image metadata", 2, self.verbose)
self.conn.execute("INSERT OR REPLACE INTO image values (?,?,?,?,?,?,?)",
- (package.md5,
- package.name,
- package.version,
- package.date,
- package.author,
- package.description,
- package.size,
+ (image.md5,
+ image.name,
+ image.version,
+ image.date,
+ image.author,
+ image.description,
+ image.size,
))
# insert data informations
- arrow("Add data metadata", 2, self.verbose)
- for key,value in package.data.items():
- self.conn.execute("INSERT OR REPLACE INTO data values (?,?,?,?)",
- (value["md5"],
- package.md5,
- key,
- value["size"]
+ arrow("Add payload metadata", 2, self.verbose)
+ for name, obj in image.payload.items():
+ self.conn.execute("INSERT OR REPLACE INTO payload values (?,?,?,?,?)",
+ (obj.md5,
+ image.md5,
+ name,
+ obj.isdir,
+ obj.size,
))
# on commit
arrow("Commit transaction to db", 1, self.verbose)
self.conn.execute("COMMIT TRANSACTION")
except Exception as e:
- raise
raise Exception("Adding metadata fail: %s" % e)
def delete(self, name, version):
diff --git a/installsystems/image.py b/installsystems/image.py
index 48c24998c7b665d7c0a027a5627a23f24e4f7433..f4900bfa58e185c9f6a17502a1afc3332e45cd95 100644
--- a/installsystems/image.py
+++ b/installsystems/image.py
@@ -16,45 +16,57 @@ import tarfile
import re
import cStringIO
import shutil
+import gzip
import installsystems.template as istemplate
import installsystems.tools as istools
from installsystems.printer import *
from installsystems.tarball import Tarball
+
class Image(object):
- '''Abstract class of images'''
+ '''
+ Abstract class of images
+ '''
- extension = ".isimage"
- extension_data = ".isdata"
format = "1"
+ extension = ".isimage"
@staticmethod
def check_image_name(buf):
- '''Check if @name is a valid image name'''
+ '''
+ Check if @name is a valid image name
+ '''
return re.match("\w+", buf) is not None
@staticmethod
def check_image_version(buf):
- '''Check if @name is a valid image version'''
+ '''
+ Check if @name is a valid image version
+ '''
return re.match("\d+", buf) is not None
+
class SourceImage(Image):
- '''Image source manipulation class'''
+ '''
+ Image source manipulation class
+ '''
@classmethod
def create(cls, path, verbose=True):
- '''Create an empty source image'''
+ '''
+ Create an empty source image
+ '''
# check local repository
if istools.pathtype(path) != "file":
raise NotImplementedError("SourceImage must be local")
# main path
parser_path = os.path.join(path, "parser")
setup_path = os.path.join(path, "setup")
- data_path = os.path.join(path, "data")
+ payload_path = os.path.join(path, "payload")
# create base directories
arrow("Creating base directories", 1, verbose)
try:
- for d in (path, parser_path, setup_path, data_path):
+ for d in (path, parser_path, setup_path, payload_path):
if not os.path.exists(d) or not os.path.isdir(d):
os.mkdir(d)
except Exception as e:
@@ -80,8 +92,7 @@ class SourceImage(Image):
os.umask(umask)
for dpath in (parser_path, setup_path):
for f in os.listdir(dpath):
- pf = os.path.join(dpath, f)
- os.chmod(pf, 0777 & ~umask)
+ istools.chrights(os.path.join(dpath, f), mode=0777 & ~umask)
except Exception as e:
raise Exception("Unable to set rights on %s: %s" % (pf, e))
return cls(path, verbose)
@@ -94,109 +105,145 @@ class SourceImage(Image):
self.base_path = path
self.parser_path = os.path.join(path, "parser")
self.setup_path = os.path.join(path, "setup")
- self.data_path = os.path.join(path, "data")
+ self.payload_path = os.path.join(path, "payload")
self.verbose = verbose
- self.validate_source_image()
+ self.validate_source_files()
self.description = self.parse_description()
+ # script tarball path
+ self.image_name = "%s-%s%s" % (self.description["name"],
+ self.description["version"],
+ self.extension)
- def validate_source_image(self):
- '''Check if we are a valid SourceImage'''
- for d in (self.base_path, self.parser_path, self.setup_path, self.data_path):
+ def validate_source_files(self):
+ '''
+ Check if we are a valid SourceImage directories
+ '''
+ for d in (self.base_path, self.parser_path, self.setup_path, self.payload_path):
if not os.path.exists(d):
raise Exception("Missing directory: %s" % d)
if not os.path.isdir(d):
raise Exception("Not a directory: %s" % d)
if not os.access(d, os.R_OK|os.X_OK):
raise Exception("Unable to access to %s" % d)
+ if not os.path.exists(os.path.join(self.base_path, "description")):
+ raise Exception("No description file")
def build(self, overwrite=False):
- '''Create packaged image'''
- # compute script tarball paths
- tarpath = os.path.join(self.base_path,
- "%s-%s%s" % (self.description["name"],
- self.description["version"],
- self.extension))
+ '''
+ Create packaged image
+ '''
# check if free to create script tarball
- if os.path.exists(tarpath) and overwrite == False:
+ if os.path.exists(self.image_name) and overwrite == False:
raise Exception("Tarball already exists. Remove it before")
- # Create data tarballs
- self.create_data_tarballs()
- # generate description.json
- jdesc = self.generate_json_description()
+ # Create payload files
+ payloads = self._create_payloads()
+ # generate a JSON description
+ jdesc = self.generate_json_description(payloads)
# creating scripts tarball
- arrow("Creating scripts tarball", 1, self.verbose)
- arrow("Name %s" % os.path.relpath(tarpath), 2, self.verbose)
+ self._create_image(jdesc)
+
+ def _create_image(self, description):
+ '''
+ Create a script tarball in current directory
+ '''
+ # create tarball
+ arrow("Creating image tarball", 1, self.verbose)
+ arrow("Name %s" % self.image_name, 2, self.verbose)
try:
- tarball = Tarball.open(tarpath, mode="w:gz", dereference=True)
+ tarball = Tarball.open(self.image_name, mode="w:gz", dereference=True)
except Exception as e:
- raise Exception("Unable to create tarball %s: %s" % (tarpath, e))
+ raise Exception("Unable to create tarball %s: %s" % (self.image_name, e))
# add .description.json
- arrow("Add .description.json", 2, self.verbose)
- tarball.add_str("description.json", jdesc, tarfile.REGTYPE, 0444)
+ arrow("Add description.json", 2, self.verbose)
+ tarball.add_str("description.json", description, tarfile.REGTYPE, 0444)
# add .format
- arrow("Add .format", 2, self.verbose)
+ arrow("Add format", 2, self.verbose)
tarball.add_str("format", self.format, tarfile.REGTYPE, 0444)
# add parser scripts
arrow("Add parser scripts", 2, self.verbose)
tarball.add(self.parser_path, arcname="parser",
- recursive=True, filter=self.tar_scripts_filter)
+ recursive=True, filter=self._tar_scripts_filter)
# add setup scripts
arrow("Add setup scripts", 2, self.verbose)
tarball.add(self.setup_path, arcname="setup",
- recursive=True, filter=self.tar_scripts_filter)
+ recursive=True, filter=self._tar_scripts_filter)
# closing tarball file
tarball.close()
- @property
- def data_tarballs(self):
- '''List all data tarballs in data directory'''
- databalls = dict()
- for dname in os.listdir(self.data_path):
- filename = "%s-%s-%s%s" % (self.description["name"],
- self.description["version"],
- dname,
- self.extension_data)
- databalls[dname] = filename
- return databalls
-
- def create_data_tarballs(self):
+ def _create_payloads(self):
'''
- Create all data tarballs in data directory
+ Create all data payloads in current directory
Doesn't compute md5 during creation because tarball can
be created manually
'''
- arrow("Creating data tarballs", 1, self.verbose)
- # build list of data tarball candidate
- candidates = self.data_tarballs
+ arrow("Creating payloads", 1, self.verbose)
+ # build list of payload files
+ candidates = os.listdir(self.payload_path)
if len(candidates) == 0:
- arrow("No data tarball", 2, self.verbose)
- return
- # create tarballs
- for (dn, df) in candidates.items():
- source_path = os.path.join(self.data_path, dn)
- dest_path = os.path.join(self.base_path, df)
+ arrow("No payload", 2, self.verbose)
+ return []
+ # create payload files
+ l_l = []
+ for pay in candidates:
+ source_path = os.path.join(self.payload_path, pay)
+ dest_path = "%s-%s-%s%s" % (self.description["name"],
+ self.description["version"],
+ pay,
+ Payload.extension)
+ source_stat = os.stat(source_path)
+ isdir = stat.S_ISDIR(source_stat.st_mode)
if os.path.exists(dest_path):
- arrow("Tarball %s already exists." % df, 2, self.verbose)
+ arrow("Payload %s already exists" % dest_path, 2, self.verbose)
else:
- arrow("Creating tarball %s" % df, 2, self.verbose)
- self.create_data_tarball(dest_path, source_path)
+ arrow("Creating payload %s" % dest_path, 2, self.verbose)
+ if isdir:
+ self._create_payload_tarball(dest_path, source_path)
+ else:
+ self._create_payload_file(dest_path, source_path)
+ # create payload object
+ payobj = Payload(pay, dest_path, isdir=isdir)
+ payobj.uid = source_stat.st_uid
+ payobj.gid = source_stat.st_gid
+ payobj.mode = stat.S_IMODE(source_stat.st_mode)
+ payobj.mtime = source_stat.st_mtime
+ l_l.append(payobj)
+ return l_l
- def create_data_tarball(self, tar_path, data_path):
- '''Create a data tarball'''
+ def _create_payload_tarball(self, tar_path, data_path):
+ '''
+ Create a payload tarball
+ This is needed by payload directory
+ '''
# compute dname to set as a base directory
dname = os.path.basename(data_path)
- # not derefence for directory. Verbatim copy.
- ddref = False if os.path.isdir(data_path) else True
try:
# Tarballing
- tarball = Tarball.open(tar_path, "w:gz", dereference=ddref)
+ tarball = Tarball.open(tar_path, "w:gz", dereference=False)
tarball.add(data_path, arcname="/", recursive=True)
tarball.close()
except Exception as e:
- raise Exception("Unable to create data tarball %s: %s" % (tar_path, e))
+ raise Exception("Unable to create payload tarball %s: %s" % (tar_path, e))
+
+ def _create_payload_file(self, dest, source):
+ '''
+ Create a payload file
+ Only gzipping it
+ '''
+ fsource = istools.uopen(source)
+ # open file not done in GzipFile, to escape writing of filename
+ # in gzip file. This change md5.
+ fdest = open(dest, "wb")
+ fdest = gzip.GzipFile(filename=os.path.basename(source),
+ fileobj=fdest,
+ mtime=os.stat(source).st_mtime)
+ istools.copyfileobj(fsource, fdest)
+ fsource.close()
+ fdest.close()
- def tar_scripts_filter(self, tinfo):
- '''Filter files which can be included in scripts tarball'''
+ def _tar_scripts_filter(self, tinfo):
+ '''
+ Filter files which can be included in scripts tarball
+ '''
if not tinfo.name in ("parser", "setup") and os.path.splitext(tinfo.name)[1] != ".py":
return None
tinfo.mode = 0755
@@ -204,26 +251,28 @@ class SourceImage(Image):
tinfo.uname = tinfo.gname = "root"
return tinfo
- def generate_json_description(self):
- '''Generate a JSON description file'''
+ def generate_json_description(self, payloads):
+ '''
+ Generate a JSON description file
+ '''
arrow("Generating JSON description", 1, self.verbose)
# copy description
desc = self.description.copy()
# timestamp image
arrow("Timestamping", 2, self.verbose)
desc["date"] = int(time.time())
- # append data tarballs info
- desc["data"] = dict()
- for (dn, df) in self.data_tarballs.items():
- arrow("Compute MD5 of %s" % df, 2, self.verbose)
- tb_path = os.path.join(self.base_path, df)
- desc["data"][dn] = { "size": os.path.getsize(tb_path),
- "md5": istools.md5sum(tb_path) }
+ # append payload infos
+ arrow("Checksumming", 2, self.verbose)
+ desc["payload"] = {}
+ for payload in payloads:
+ desc["payload"][payload.name] = payload.info
# serialize
return json.dumps(desc)
def parse_description(self):
- '''Raise an exception is description file is invalid and return vars to include'''
+ '''
+ Raise an exception is description file is invalid and return vars to include
+ '''
arrow("Parsing description", 1, self.verbose)
d = dict()
try:
@@ -233,12 +282,14 @@ class SourceImage(Image):
for n in ("name","version", "description", "author"):
d[n] = cp.get("image", n)
except Exception as e:
- raise Exception("Invalid description: %s" % e)
+ raise Exception("Bad description: %s" % e)
return d
class PackageImage(Image):
- '''Packaged image manipulation class'''
+ '''
+ Packaged image manipulation class
+ '''
def __init__(self, path, md5name=False, verbose=True):
Image.__init__(self)
@@ -257,25 +308,43 @@ class PackageImage(Image):
memfile.seek(0)
self._tarball = Tarball.open(fileobj=memfile, mode='r:gz')
self._metadata = self.read_metadata()
+ # build payloads
+ self.payload = {}
+ for pname, pval in self._metadata["payload"].items():
+ if self.md5name:
+ ppath = os.path.join(self.base_path,
+ self._metadata["payload"][pname]["md5"])
+ else:
+ ppath = os.path.join(self.base_path,
+ "%s-%s%s" % (self.id, pname, Payload.extension))
+ self.payload[pname] = Payload(pname, ppath, **pval)
def __getattr__(self, name):
- """Give direct access to description field"""
+ '''
+ Give direct access to description field
+ '''
if name in self._metadata:
return self._metadata[name]
raise AttributeError
@property
def id(self):
- '''Return image versionned name / id'''
- return "%s-%s" % (self._metadata["name"], self._metadata["version"])
+ '''
+ Return image versionned name / id
+ '''
+ return "%s-%s" % (self.name, self.version)
@property
def filename(self):
- '''Return image filename'''
+ '''
+ Return image filename
+ '''
return "%s%s" % (self.id, self.extension)
def read_metadata(self):
- '''Parse tarball and return metadata dict'''
+ '''
+ Parse tarball and return metadata dict
+ '''
# extract metadata
arrow("Read tarball metadata", 1, self.verbose)
img_format = self._tarball.get_str("format")
@@ -285,7 +354,7 @@ class PackageImage(Image):
if img_format != self.format:
raise Exception("Invalid tarball image format")
# check description
- arrow("Read description file", 2, self.verbose)
+ arrow("Read image description", 2, self.verbose)
try:
desc = json.loads(img_desc)
except Exception as e:
@@ -293,54 +362,36 @@ class PackageImage(Image):
# FIXME: we should check valid information here
return desc
- def data_path(self, name):
- '''Return a data filename from its internal name'''
- if self.md5name:
- return os.path.join(self.base_path, self._metadata["data"][name]["md5"])
- return os.path.join(self.base_path, "%s-%s%s" % (self.id, name, self.extension_data))
-
- @property
- def tarballs(self):
- '''List path of all related tarballs'''
- d_d = {}
- name = os.path.join(self.base_path, self.md5) if self.md5name else self.path
- d_d[name] = {"md5": self.md5, "size": self.size}
- for key, value in self._metadata["data"].items():
- d_d[self.data_path(key)] = {"md5": value["md5"], "size": value["size"]}
- return d_d
-
def check(self, message="Check MD5"):
- '''Check md5 and size of tarballs are correct'''
+ '''
+ Check md5 and size of tarballs are correct
+ '''
arrow(message, 1, self.verbose)
- # open /dev/null
- dn = open("/dev/null", "w")
- for key,value in self.tarballs.items():
- arrow(os.path.basename(key), 2, self.verbose)
- # open tarball
- tfo = istools.uopen(key)
- # compute sum and md5 using copy function
- size, md5 = istools.copyfileobj(tfo ,dn)
- # close tarball fo
- tfo.close()
- # check md5
- if md5 != value["md5"]:
- raise Exception("Invalid md5: %s" % key)
- # check size
- if size != value["size"]:
- raise Exception("Invalid size: %s" % key)
- dn.close()
+ # check image
+ if self.md5 != istools.md5sum(self.path):
+ raise Exception("Invalid MD5 of image %s" % self.name)
+ # check payloads
+ for pay_name, pay_obj in self.payload.items():
+ arrow(pay_name, 2, self.verbose)
+ pay_obj.check()
def run_parser(self, gl):
- '''Run parser scripts'''
+ '''
+ Run parser scripts
+ '''
self._run_scripts(gl, "parser")
def run_setup(self, gl):
- '''Run setup scripts'''
+ '''
+ Run setup scripts
+ '''
gl["image"] = self
self._run_scripts(gl, "setup")
def _run_scripts(self, gl, directory):
- '''Run scripts in a tarball directory'''
+ '''
+ Run scripts in a tarball directory
+ '''
arrow("Run %s" % directory, 1, self.verbose)
# get list of parser scripts
l_scripts = self._tarball.getnames("%s/.*\.py" % directory)
@@ -357,31 +408,205 @@ class PackageImage(Image):
try:
exec(s_scripts, gl, dict())
except Exception as e:
- raise
raise Exception("Execution script %s fail: %s" %
(os.path.basename(n_scripts), e))
- def extractdata(self, dataname, target, filelist=None):
- '''Extract a data tarball into target'''
- # check if dataname exists
- if dataname not in self._metadata["data"].keys():
- raise Exception("No such data: %s" % dataname)
- # tarball info
- tinfo = self._metadata["data"][dataname]
- # build data tar paths
- path = self.data_path(dataname)
+
+class Payload(object):
+ '''
+ Payload class represents a payload object
+ '''
+ extension = ".isdata"
+ legit_attr = ('isdir', 'md5', 'size', 'uid', 'gid', 'mode', 'mtime')
+
+ def __init__(self, name, path, **kwargs):
+ object.__setattr__(self, "name", name)
+ object.__setattr__(self, "path", path)
+ # register legit param
+ for attr in self.legit_attr:
+ setattr(self, attr, None)
+ # set all named param
+ for kwarg in kwargs:
+ if hasattr(self, kwarg):
+ setattr(self, kwarg, kwargs[kwarg])
+
+ def __getattr__(self, name):
+ # get all value with an understance as if there is no underscore
+ if hasattr(self, "_%s" % name):
+ return getattr(self, "_%s" % name)
+ raise AttributeError
+
+ def __setattr__(self, name, value):
+ # set all value which exists have no underscore, but undesrcore exists
+ if name in self.legit_attr:
+ object.__setattr__(self, "_%s" % name, value)
+ else:
+ object.__setattr__(self, name, value)
+
+ def checksummize(self):
+ '''
+ Fill missing md5/size about payload
+ '''
+ fileobj = istools.uopen(self.path)
+ size, md5 = istools.copyfileobj(fileobj, None)
+ if self._size is None:
+ self._size = size
+ if self._md5 is None:
+ self._md5 = md5
+
+ @property
+ def md5(self):
+ '''
+ Return md5 of payload
+ '''
+ if self._md5 is None:
+ self.checksummize()
+ return self._md5
+
+ @property
+ def size(self):
+ '''
+ Return size of payload
+ '''
+ if self._size is None:
+ self.checksummize()
+ return self._size
+
+ @property
+ def uid(self):
+ '''
+ Return uid of owner of orginal payload
+ '''
+ return self._uid if self._uid is not None else 0
+
+ @property
+ def gid(self):
+ '''
+ Return gid of owner of orginal payload
+ '''
+ return self._gid if self._gid is not None else 0
+
+ @property
+ def mode(self):
+ '''
+ Return mode of orginal payload
+ '''
+ if self._mode is not None:
+ return self._mode
+ else:
+ umask = os.umask(0)
+ os.umask(umask)
+ return 0666 & ~umask
+
+ @property
+ def mtime(self):
+ '''
+ Return last modification time of orginal payload
+ '''
+ return self._mtime if self._mtime is not None else time.time()
+
+ @property
+ def info(self):
+ '''
+ return a dict of info about current payload
+ '''
+ return {"md5": self.md5,
+ "size": self.size,
+ "isdir": self.isdir,
+ "uid": self.uid,
+ "gid": self.gid,
+ "mode": self.mode,
+ "mtime": self.mtime}
+
+ def check(self):
+ '''
+ Check that path correspond to current md5 and size
+ '''
+ if self._size is None or self._md5 is None:
+ debug("Check is called on payload with nothing to check")
+ return True
+ fileobj = istools.uopen(self.path)
+ size, md5 = istools.copyfileobj(fileobj, None)
+ if self._size != size:
+ raise Exception("Invalid size of payload %s" % self.name)
+ if self._md5 != md5:
+ raise Exception("Invalid MD5 of payload %s" % self._md5)
+
+ def extract(self, dest, force=False, filelist=None):
+ '''
+ Extract payload into dest
+ filelist is a filter of file in tarball
+ force will overwrite existing file if exists
+ '''
+ if self.isdir:
+ self.extract_tar(dest, force=force, filelist=filelist)
+ else:
+ self.extract_file(dest, force=force)
+
+ def extract_tar(self, dest, force=False, filelist=None):
+ '''
+ Extract a payload which is a tarball.
+ This is used mainly to extract payload from a directory
+ '''
+ # check validity of dest
+ if os.path.exists(dest):
+ if not os.path.isdir(dest):
+ raise Exception("Destination %s is not a directory" % dest)
+ if not force and len(os.listdir(dest)) > 0:
+ raise Exception("Directory %s is not empty (need force)" % dest)
+ else:
+ os.mkdir(dest)
+ # try to open payload file
try:
- fo = istools.uopen(path)
+ fo = istools.uopen(self.path)
except Exception as e:
- raise Exception("Unable to open data tarball %s" % path)
+ raise Exception("Unable to open payload file %s" % self.path)
+ # try to open tarball on payload
try:
- # create tar object
t = Tarball.open(fileobj=fo, mode="r|gz")
except Exception as e:
- raise Exception("Invalid data tarball: %s" % e)
+ raise Exception("Invalid payload tarball: %s" % e)
# filter on file to extact
- members = None if filelist is None else [ t.gettarinfo(name) for name in filelist ]
+ members = (None if filelist is None
+ else [ t.gettarinfo(name) for name in filelist ])
try:
- t.extractall(target, members)
+ t.extractall(dest, members)
except Exception as e:
raise Exception("Extracting failed: %s" % e)
+ # closing fo
+ t.close()
+ fo.close()
+
+ def extract_file(self, dest, force=False):
+ '''
+ Copy a payload directly to a file
+ Check md5 on the fly
+ '''
+ # if dest is a directory try to create file inside
+ if os.path.isdir(dest):
+ dest = os.path.join(dest, self.name)
+ # check validity of dest
+ if os.path.exists(dest):
+ if not os.path.isfile(dest):
+ raise Exception("Destination %s is not a file" % dest)
+ if not force:
+ raise Exception("File %s already exists" % dest)
+ # opening destination
+ try:
+ f_dst = istools.uopen(dest, "wb")
+ except Exception as e:
+ raise Exception("Unable to open destination file %s" % dest)
+ # try to open payload file
+ try:
+ f_gsrc = istools.uopen(self.path)
+ f_src = gzip.GzipFile(fileobj=f_gsrc)
+ except Exception as e:
+ raise Exception("Unable to open payload file %s" % self.path)
+ # launch copy
+ size, md5 = istools.copyfileobj(f_src, f_dst)
+ # closing fo
+ f_dst.close()
+ f_gsrc.close()
+ f_src.close()
+ # settings file orginal rights
+ istools.chrights(dest, self.uid, self.gid, self.mode, self.mtime)
diff --git a/installsystems/repository.py b/installsystems/repository.py
index b22fb419d956cdef3055054742837c3eb6df8d31..1b0557483f0d5f463883d32a06a3a5a65b42ab3c 100644
--- a/installsystems/repository.py
+++ b/installsystems/repository.py
@@ -77,30 +77,33 @@ class Repository(object):
raise Exception("Read last file failed: %s" % e)
return 0
- def add(self, package):
+ def add(self, image):
'''Add a packaged image to repository'''
# check local repository
if istools.pathtype(self.config.path) != "file":
raise NotImplementedError("Repository addition must be local")
# checking data tarballs md5 before copy
- package.check("Check tarballs before copy")
+ image.check("Check image and payload before copy")
# adding file to repository
- arrow("Copying files", 1, self.verbose)
- for src,value in package.tarballs.items():
- dest = os.path.join(self.config.path, value["md5"])
- basesrc = os.path.basename(src)
+ arrow("Copying images and payload", 1, self.verbose)
+ for obj in [ image ] + image.payload.values():
+ dest = os.path.join(self.config.path, obj.md5)
+ basesrc = os.path.basename(obj.path)
if os.path.exists(dest):
arrow("Skipping %s: already exists" % basesrc, 2, self.verbose)
else:
- arrow("Adding %s (%s)" % (basesrc, value["md5"]), 2, self.verbose)
- istools.copy(src, dest, self.config.uid, self.config.gid, self.config.fmod)
- # copy is done. create a package inside repo
- r_package = PackageImage(os.path.join(self.config.path, package.md5),
+ arrow("Adding %s (%s)" % (basesrc, obj.md5), 2, self.verbose)
+ istools.copy(obj.path, dest,
+ self.config.uid, self.config.gid, self.config.fmod)
+ # copy is done. create a image inside repo
+ r_image = PackageImage(os.path.join(self.config.path, image.md5),
md5name=True, verbose=self.verbose)
- # checking data tarballs md5 after copy
- r_package.check("Check tarballs after copy")
+ # checking must be done with original md5
+ r_image.md5 = image.md5
+ # checking image and payload after copy
+ r_image.check("Check image and payload after copy")
# add description to db
- self.db.add(r_package)
+ self.db.add(r_image)
# update last file
self.update_last()
@@ -137,7 +140,7 @@ class Repository(object):
return self.db.ask("select name,version from image where name = ? and version = ? limit 1", (name,version)).fetchone() is not None
def get(self, name, version):
- '''return a package from a name and version of pakage'''
+ '''return a image from a name and version of pakage'''
# get file md5 from db
r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
(name,version)).fetchone()
diff --git a/installsystems/tarball.py b/installsystems/tarball.py
index 1c0fe0ff4dedaf9ec30216000ba42a943420dfb0..cf8d52c01b727272c102f9abcf34ec8340882c3a 100644
--- a/installsystems/tarball.py
+++ b/installsystems/tarball.py
@@ -36,3 +36,10 @@ class Tarball(tarfile.TarFile):
else:
return [ tpname for tpname in lorig
if re.match(reg_pattern, tpname) ]
+
+ def size(self):
+ '''Return real (uncompressed) size of the tarball'''
+ total_sz = 0
+ for ti in self.getmembers():
+ total_sz += ti.size
+ return total_sz
diff --git a/installsystems/template.py b/installsystems/template.py
index 748ffd774e02698d1cb51573654e337b890dece8..af960e5801d24f745e5ca1b8064ce389a91581da 100644
--- a/installsystems/template.py
+++ b/installsystems/template.py
@@ -24,7 +24,7 @@ setup = """# -*- python -*-
print "hostname: %s" % args.hostname
-image.extractdata("rootfs", args.target)
+image.payload["rootfs"].extract(args.target)
# vim:set ts=2 sw=2 noet:
"""
@@ -39,9 +39,10 @@ CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
size INTEGER NOT NULL,
UNIQUE(name, version));
-CREATE TABLE data (md5 TEXT NOT NULL,
- image_md5 TEXT NOT NULL REFERENCES image(md5),
- name TEXT NOT NULL,
- size INTEGER NOT NULL,
- PRIMARY KEY(md5, image_md5));
+CREATE TABLE payload (md5 TEXT NOT NULL,
+ image_md5 TEXT NOT NULL REFERENCES image(md5),
+ name TEXT NOT NULL,
+ isdir INTEGER NOT NULL,
+ size INTEGER NOT NULL,
+ PRIMARY KEY(md5, image_md5));
"""
diff --git a/installsystems/tools.py b/installsystems/tools.py
index f42f85c78a1cfe3bc012c25d0788d0ca970b9138..d45d04237c67c14a1625a8954b991bbfd8a7e607 100644
--- a/installsystems/tools.py
+++ b/installsystems/tools.py
@@ -37,7 +37,8 @@ def copyfileobj(sfile, dfile):
break
f_len += buf_len
f_sum.update(buf)
- dfile.write(buf)
+ if dfile is not None:
+ dfile.write(buf)
return (f_len , f_sum.hexdigest())
def copy(source, destination, uid=None, gid=None, mode=None, timeout=None):
@@ -67,7 +68,7 @@ def mkdir(path, uid=None, gid=None, mode=None):
os.mkdir(path)
chrights(path, uid, gid, mode)
-def chrights(path, uid=None, gid=None, mode=None):
+def chrights(path, uid=None, gid=None, mode=None, mtime=None):
'''Set rights on a file'''
if uid is not None:
os.chown(path, uid, -1)
@@ -75,6 +76,8 @@ def chrights(path, uid=None, gid=None, mode=None):
os.chown(path, -1, gid)
if mode is not None:
os.chmod(path, mode)
+ if mtime is not None:
+ os.utime(path, (mtime, mtime))
def pathtype(path):
'''Return path type. This is usefull to know what kind of path is given'''
@@ -103,14 +106,26 @@ def abspath(path):
else:
return None
-def uopen(path):
+def uopen(path, mode="rb"):
'''Universal Open
Create a file-like object to a file which can be remote
'''
ftype = pathtype(path)
if ftype == "file":
- return open(path, "r")
+ return open(path, mode)
elif ftype == "http" or ftype == "ftp":
return urllib2.urlopen(path)
else:
raise NotImplementedError
+
+def getsize(path):
+ '''Get size of a path. Recurse if directory'''
+ total_sz = os.path.getsize(path)
+ if os.path.isdir(path):
+ for root, dirs, files in os.walk(path):
+ for filename in dirs + files:
+ filepath = os.path.join(root, filename)
+ filestat = os.lstat(filepath)
+ if stat.S_ISDIR(filestat.st_mode) or stat.S_ISREG(filestat.st_mode):
+ total_sz += filestat.st_size
+ return total_sz