Newer
Older
# -*- python -*-
# -*- coding: utf-8 -*-
# Started 10/05/2011 by Seblu <seblu@seblu.net>
'''
Image stuff
'''
import os
import stat
import time
import json
import ConfigParser
import subprocess
import gzipstream #until python support gzip not seekable
import installsystems.template as istemplate
import installsystems.tools as istools
from installsystems.tools import PipeFile
from installsystems.tarball import Tarball
format = "1"
if re.match("^[-_\w]+$", buf) is None:
raise Exception("Invalid image name %s" % buf)
@staticmethod
def check_image_version(buf):
if re.match("^\d+$", buf) is None:
raise Exception("Invalid image version %s" % buf)
if not istools.isfile(path):
raise NotImplementedError("SourceImage must be local")
# main path
parser_path = os.path.join(path, "parser")
setup_path = os.path.join(path, "setup")
if not os.path.exists(d) or not os.path.isdir(d):
os.mkdir(d)
raise Exception("Unable to create directory: %s: %s" % (d, e))
try:
# create description example from template
open(os.path.join(path, "description"), "w").write(istemplate.description)
# create changelog example from template
arrow("Creating description example")
open(os.path.join(path, "changelog"), "w").write(istemplate.changelog)
open(os.path.join(parser_path, "01-parser.py"), "w").write(istemplate.parser)
open(os.path.join(setup_path, "01-setup.py"), "w").write(istemplate.setup)
except Exception as e:
raise Exception("Unable to example file: %s" % e)
try:
# setting rights on files in setup and parser
umask = os.umask(0)
os.umask(umask)
for dpath in (parser_path, setup_path):
for f in os.listdir(dpath):
except Exception as e:
raise Exception("Unable to set rights on %s: %s" % (pf, e))
if not istools.isfile(path):
raise NotImplementedError("SourceImage must be local")
self.base_path = path
self.parser_path = os.path.join(path, "parser")
self.setup_path = os.path.join(path, "setup")
self.payload_path = os.path.join(path, "payload")
self.validate_source_files()
self.description = self.parse_description()
# script tarball path
self.image_name = "%s-%s%s" % (self.description["name"],
self.description["version"],
self.extension)
def validate_source_files(self):
'''
Check if we are a valid SourceImage directories
'''
for d in (self.base_path, self.parser_path, self.setup_path, self.payload_path):
if not os.path.exists(d):
raise Exception("Missing directory: %s" % d)
if not os.path.isdir(d):
raise Exception("Not a directory: %s" % d)
if not os.access(d, os.R_OK|os.X_OK):
raise Exception("Unable to access to %s" % d)
if not os.path.exists(os.path.join(self.base_path, "description")):
raise Exception("No description file")
def build(self, force=False, check=True):
# check if free to create script tarball
if os.path.exists(self.image_name) and force == False:
raise Exception("Tarball already exists. Remove it before")
# Check python file
if check:
self.check_scripts(self.parser_path)
self.check_scripts(self.setup_path)
# generate a JSON description
jdesc = self.generate_json_description(payloads)
'''
Create a script tarball in current directory
'''
# create tarball
arrow("Creating image tarball")
arrowlevel(1)
arrow("Name %s" % self.image_name)
tarball = Tarball.open(self.image_name, mode="w:gz", dereference=True)
raise Exception("Unable to create tarball %s: %s" % (self.image_name, e))
tarball.add_str("description.json", jdescription, tarfile.REGTYPE, 0444)
tarball.add_str("changelog", self.changelog.verbatim, tarfile.REGTYPE, 0444)
tarball.add_str("format", self.format, tarfile.REGTYPE, 0444)
self._add_scripts(tarball, self.parser_path)
self._add_scripts(tarball, self.setup_path)
# closing tarball file
tarball.close()
# build list of payload files
candidates = os.listdir(self.payload_path)
return []
# create payload files
l_l = []
for pay in candidates:
source_path = os.path.join(self.payload_path, pay)
dest_path = "%s-%s-%s%s" % (self.description["name"],
self.description["version"],
pay,
Payload.extension)
source_stat = os.stat(source_path)
isdir = stat.S_ISDIR(source_stat.st_mode)
if isdir:
self._create_payload_tarball(dest_path, source_path)
else:
self._create_payload_file(dest_path, source_path)
# create payload object
payobj = Payload(pay, dest_path, isdir=isdir)
payobj.uid = source_stat.st_uid
payobj.gid = source_stat.st_gid
payobj.mode = stat.S_IMODE(source_stat.st_mode)
payobj.mtime = source_stat.st_mtime
l_l.append(payobj)
def _create_payload_tarball(self, tar_path, data_path):
'''
Create a payload tarball
This is needed by payload directory
'''
# compute dname to set as a base directory
dname = os.path.basename(data_path)
try:
dfo = PipeFile(tar_path, "w", progressbar=True)
tarball = Tarball.open(fileobj=dfo, mode="w|gz", dereference=False)
tarball.add(data_path, arcname="/", recursive=True,
filter=self._create_payload_tarball_filter)
raise Exception("Unable to create payload tarball %s: %s" % (tar_path, e))
def _create_payload_tarball_filter(self, tarinfo):
'''
Have the same behaviour as --numeric-owner on gnu tar
Remove string name and group to escape weird translation
'''
tarinfo.uname = ""
tarinfo.gname = ""
return tarinfo
def _create_payload_file(self, dest, source):
'''
Create a payload file
Only gzipping it
'''
fsource = PipeFile(source, "r", progressbar=True)
# open file not done in GzipFile, to escape writing of filename
# in gzip file. This change md5.
fdest = open(dest, "wb")
fdest = gzip.GzipFile(filename=os.path.basename(source),
fileobj=fdest,
mtime=os.stat(source).st_mtime)
def _add_scripts(self, tarball, directory):
Add scripts inside a directory into a tarball
basedirectory = os.path.basename(directory)
arrow("Add %s scripts" % basedirectory)
arrowlevel(1)
# adding base directory
ti = tarball.gettarinfo(directory, arcname=basedirectory)
ti.mode = 0755
ti.uid = ti.gid = 0
ti.uname = ti.gname = "root"
tarball.addfile(ti)
# adding each file
for fi in os.listdir(directory):
# check name
if not re.match("\d+-.*\.py$", fi):
debug("%s skipped: invalid name" % fi)
continue
# adding file
ti = tarball.gettarinfo(fp, arcname=os.path.join(basedirectory, fi))
ti.mode = 0755
ti.uid = ti.gid = 0
ti.uname = ti.gname = "root"
arrow("%s added" % fi)
arrowlevel(-1)
'''
Check if scripts inside a directory can be compiled
'''
basedirectory = os.path.basename(directory)
arrow("Checking %s scripts" % basedirectory)
arrowlevel(1)
# checking each file
for fi in os.listdir(directory):
# check name
if not re.match("\d+-.*\.py$", fi):
debug("%s skipped: invalid name" % fi)
continue
# compiling file
fs = open(os.path.join(directory, fi), "rb").read()
compile(fs, fi, mode="exec")
arrow(fi)
arrowlevel(-1)
def generate_json_description(self, payloads):
'''
Generate a JSON description file
'''
arrow("Generating JSON description")
arrowlevel(1)
# copy description
desc = self.description.copy()
# timestamp image
# watermark
desc["isversion"] = installsystems.version
desc["payload"] = {}
for payload in payloads:
desc["payload"][payload.name] = payload.info
# serialize
return json.dumps(desc)
def parse_description(self):
'''
Raise an exception is description file is invalid and return vars to include
'''
d = dict()
try:
descpath = os.path.join(self.base_path, "description")
cp = ConfigParser.RawConfigParser()
cp.read(descpath)
for n in ("name","version", "description", "author"):
d[n] = cp.get("image", n)
self.check_image_name(d["name"])
self.check_image_version(d["version"])
def parse_changelog(self):
'''
Create a changelog object from a file
'''
# try to find a changelog file
try:
path = os.path.join(self.base_path, "changelog")
fo = open(path, "r")
except IOError:
return None
# we have it, we need to check everything is ok
arrow("Parsing changelog")
try:
cl = Changelog(fo.read())
except Exception as e:
raise Exception("Bad changelog: %s" % e)
return cl
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
@classmethod
def diff(cls, pkg1, pkg2):
'''
Diff two packaged images
'''
arrow("Differnce from images #y#%s v%s#R# to #r#%s v%s#R#:" % (pkg1.name,
pkg1.version,
pkg2.name,
pkg2.version))
# Extract images for diff scripts files
fromfiles = set(pkg1._tarball.getnames(re_pattern="(parser|setup)/.*"))
tofiles = set(pkg2._tarball.getnames(re_pattern="(parser|setup)/.*"))
for f in fromfiles | tofiles:
# preparing from info
if f in fromfiles:
fromfile = os.path.join(pkg1.id, f)
fromdata = pkg1._tarball.extractfile(f).readlines()
else:
fromfile = "/dev/null"
fromdata = ""
# preparing to info
if f in tofiles:
tofile = os.path.join(pkg2.id, f)
todata = pkg2._tarball.extractfile(f).readlines()
else:
tofile = "/dev/null"
todata = ""
# generate diff
for line in difflib.unified_diff(fromdata, todata,
fromfile=fromfile, tofile=tofile):
# coloring diff
if line.startswith("+"):
out("#g#%s#R#" % line, endl="")
elif line.startswith("-"):
out("#r#%s#R#" % line, endl="")
elif line.startswith("@@"):
out("#c#%s#R#" % line, endl="")
else:
out(line, endl="")
def __init__(self, path, fileobj=None, md5name=False):
'''
Initialize a package image
fileobj must be a seekable fileobj
'''
self.base_path = os.path.dirname(self.path)
# tarball are named by md5 and not by real name
self.md5name = md5name
fileobj = PipeFile(self.path, "r")
else:
fileobj = PipeFile(mode="r", fileobj=fileobj)
memfile = cStringIO.StringIO()
fileobj.consume(memfile)
# get donwloaded size and md5
self.size = fileobj.read_size
self.md5 = fileobj.md5
memfile.seek(0)
self._tarball = Tarball.open(fileobj=memfile, mode='r:gz')
except Exception as e:
raise Exception("Unable to open image %s: %s" % (path, e))
arrow("Image %s v%s loaded" % (self.name, self.version))
arrow("Author: %s" % self.author, 1)
arrow("Date: %s" % time.ctime(self.date), 1)
self.payload = {}
for pname, pval in self._metadata["payload"].items():
if self.md5name:
ppath = os.path.join(self.base_path,
self._metadata["payload"][pname]["md5"])
else:
ppath = os.path.join(self.base_path,
"%s-%s%s" % (self.id, pname, Payload.extension))
self.payload[pname] = Payload(pname, ppath, **pval)
if name in self._metadata:
return self._metadata[name]
raise AttributeError
@property
def id(self):
'''
Return image versionned name / id
'''
return "%s-%s" % (self.name, self.version)
@property
def filename(self):
return "%s%s" % (self.id, self.extension)
if img_format != self.format:
raise Exception("Invalid tarball image format")
img_desc = self._tarball.get_str("description.json")
desc.update(json.loads(img_desc))
self.check_image_name(desc["name"])
self.check_image_version(desc["version"])
# try to load changelog
try:
img_changelog = self._tarball.get_str("changelog")
desc["changelog"] = Changelog(img_changelog)
except KeyError:
pass
except Exception as e:
warn("Invalid changelog: %s" % e)
'''
Display image content
'''
out('#light##yellow#Name:#reset# %s' % self.name)
out('#light##yellow#Version:#reset# %s' % self.version)
out('#yellow#Date:#reset# %s' % time.ctime(self.date))
out('#yellow#Description:#reset# %s' % self.description)
out('#yellow#Author:#reset# %s' % self.author)
# field isversion is new in version 5. I can be absent.
try:
out('#yellow#IS version:#reset# %s' % self.isversion)
except AttributeError:
pass
if verbose:
payloads = self.payload
for payload_name in payloads:
payload = payloads[payload_name]
out(' #yellow#Date:#reset# %s' % time.ctime(payload.mtime))
out(' #yellow#Size:#reset# %s' % (istools.human_size(payload.size)))
out(' #yellow#MD5:#reset# %s' % payload.md5)
out('#light##yellow#Content:#reset#')
self._tarball.list(verbose)
# display changelog
try:
self.changelog.show(int(self.version), verbose)
except AttributeError:
pass
Download tarball from path and compare the loaded md5 and remote
fo = PipeFile(self.path, "r")
fo.consume()
fo.close()
if self.size != fo.read_size:
raise Exception("Invalid size of image %s" % self.name)
if self.md5 != fo.md5:
raise Exception("Invalid MD5 of image %s" % self.name)
# check payloads
for pay_name, pay_obj in self.payload.items():
'''
Display filename in the tarball
'''
for filename in self._tarball.getnames(glob_pattern=filename):
arrow(filename)
out(self._tarball.get_str(filename))
def download(self, directory, force=False, payload=False):
'''
Download image in directory
Doesn't use in memory image because we cannot access it
This is done to don't parasitize self._tarfile access to memfile
'''
# check if destination exists
directory = os.path.abspath(directory)
dest = os.path.join(directory, self.filename)
if not force and os.path.exists(dest):
raise Exception("Image destination already exists: %s" % dest)
debug("Downloading %s from %s" % (self.id, self.path))
# open source
fs = PipeFile(self.path, progressbar=True)
# check if announced file size is good
if fs.size is not None and self.size != fs.size:
raise Exception("Downloading image %s failed: Invalid announced size" % self.name)
# open destination
fs.consume(fd)
if self.size != fs.consumed_size:
raise Exception("Download image %s failed: Invalid size" % self.name)
if self.md5 != fs.md5:
raise Exception("Download image %s failed: Invalid MD5" % self.name)
if payload:
for payname in self.payload:
arrow("Downloading payload %s in %s" % (payname, directory))
self.payload[payname].info
self.payload[payname].download(directory, force=force)
def extract(self, directory, force=False, payload=False):
'''
Extract content of the image inside a repository
'''
# check destination
if not os.path.isdir(directory):
istools.mkdir(directory)
if not force and len(os.listdir(directory)) != 0:
raise Exception("%s is not empty" % directory)
# extract content
arrow("Extracting image in %s" % directory)
self._tarball.extractall(directory)
# launch payload extract
if payload:
for payname in self.payload:
dest = os.path.join(directory, "payload", payname)
arrow("Extracting payload %s in %s" % (payname, dest))
self.payload[payname].extract(dest, force=force)
l_scripts = self._tarball.getnames(re_pattern="%s/.*\.py" % directory)
# order matter!
l_scripts.sort()
# run scripts
arrow(os.path.basename(n_scripts))
old_level = arrowlevel(1)
except Exception as e:
raise Exception("Extracting script %s fail: %s" %
o_scripts = compile(s_scripts, n_scripts, "exec")
except Exception as e:
raise Exception("Unable to compile %s fail: %s" %
(n_scripts, e))
# define execution context
gl = {}
for k in kwargs:
gl[k] = kwargs[k]
gl["image"] = self
# execute source code
try:
exec o_scripts in gl
except Exception as e:
raise Exception("Execution script %s fail: %s" %
class Payload(object):
'''
Payload class represents a payload object
'''
extension = ".isdata"
legit_attr = ('isdir', 'md5', 'size', 'uid', 'gid', 'mode', 'mtime')
def __init__(self, name, path, **kwargs):
object.__setattr__(self, "name", name)
object.__setattr__(self, "path", path)
# register legit param
for attr in self.legit_attr:
setattr(self, attr, None)
# set all named param
for kwarg in kwargs:
# do not use hasattr which user getattr and so call md5 checksum...
if kwarg in self.legit_attr:
setattr(self, kwarg, kwargs[kwarg])
def __getattr__(self, name):
# get all value with an understance as if there is no underscore
if hasattr(self, "_%s" % name):
return getattr(self, "_%s" % name)
raise AttributeError
def __setattr__(self, name, value):
# set all value which exists have no underscore, but where undesrcore exists
if name in self.legit_attr:
object.__setattr__(self, "_%s" % name, value)
else:
object.__setattr__(self, name, value)
def checksummize(self):
'''
Fill missing md5/size about payload
'''
fileobj = PipeFile(self.path, "r")
fileobj.consume()
fileobj.close()
self._size = fileobj.read_size
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
@property
def md5(self):
'''
Return md5 of payload
'''
if self._md5 is None:
self.checksummize()
return self._md5
@property
def size(self):
'''
Return size of payload
'''
if self._size is None:
self.checksummize()
return self._size
@property
def uid(self):
'''
Return uid of owner of orginal payload
'''
return self._uid if self._uid is not None else 0
@property
def gid(self):
'''
Return gid of owner of orginal payload
'''
return self._gid if self._gid is not None else 0
@property
def mode(self):
'''
Return mode of orginal payload
'''
if self._mode is not None:
return self._mode
else:
umask = os.umask(0)
os.umask(umask)
return 0666 & ~umask
@property
def filename(self):
'''
Return the filename of the original payload
'''
return "%s%s" % (self.name, self.extension)
@property
def mtime(self):
'''
Return last modification time of orginal payload
'''
return self._mtime if self._mtime is not None else time.time()
@property
def info(self):
'''
Return a dict of info about current payload
This info will be inserted in descript.json
Auto calculated info like name and filename must not be here
"size": self.size,
"isdir": self.isdir,
"uid": self.uid,
"gid": self.gid,
"mode": self.mode,
"mtime": self.mtime}
def check(self):
'''
Check that path correspond to current md5 and size
'''
if self._size is None or self._md5 is None:
debug("Check is called on payload with nothing to check")
return True
fileobj = PipeFile(self.path, "r")
fileobj.consume()
fileobj.close()
if self._size != fileobj.read_size:
if self._md5 != fileobj.md5:
def download(self, dest, force=False):
'''
Download payload in directory
'''
# if dest is a directory try to create file inside
if os.path.isdir(dest):
dest = os.path.join(dest, self.filename)
# try to create leading directories
elif not os.path.exists(os.path.dirname(dest)):
istools.mkdir(os.path.dirname(dest))
# check validity of dest
if os.path.exists(dest):
if os.path.isdir(dest):
raise Exception("Destination %s is a directory" % dest)
if not force:
raise Exception("File %s already exists" % dest)
# Open remote file
debug("Downloading %s from %s" % (self.name, self.path))
fs = PipeFile(self.path, progressbar=True)
# check if announced file size is good
if fs.size is not None and self.size != fs.size:
raise Exception("Downloading payload %s failed: Invalid announced size" % self.name)
fs.consume(fd)
# checking download size
if self.size != fs.read_size:
raise Exception("Downloading payload %s failed: Invalid size" % self.name)
if self.md5 != fs.md5:
raise Exception("Downloading payload %s failed: Invalid MD5" % self.name)
def extract(self, dest, force=False, filelist=None):
'''
Extract payload into dest
filelist is a filter of file in tarball
force will overwrite existing file if exists
'''
if self.isdir:
self.extract_tar(dest, force=force, filelist=filelist)
else:
self.extract_file(dest, force=force)
def extract_tar(self, dest, force=False, filelist=None):
'''
Extract a payload which is a tarball.
This is used mainly to extract payload from a directory
'''
# check validity of dest
if os.path.exists(dest):
if not os.path.isdir(dest):
raise Exception("Destination %s is not a directory" % dest)
if not force and len(os.listdir(dest)) > 0:
raise Exception("Directory %s is not empty (need force)" % dest)
else:
fo = PipeFile(self.path, progressbar=True)
# check if announced file size is good
if fo.size is not None and self.size != fo.size:
raise Exception("Invalid announced size on payload %s" % self.path)
t = Tarball.open(fileobj=fo, mode="r|gz", ignore_zeros=True)
members = (None if filelist is None
else [ t.gettarinfo(name) for name in filelist ])
raise Exception("Extracting failed: %s" % e)
# checking download size
if self.size != fo.read_size:
raise Exception("Downloading payload %s failed: Invalid size" % self.name)
if self.md5 != fo.md5:
raise Exception("Downloading payload %s failed: Invalid MD5" % self.name)
def extract_file(self, dest, force=False):
'''
Copy a payload directly to a file
Check md5 on the fly
'''
# if dest is a directory try to create file inside
if os.path.isdir(dest):
dest = os.path.join(dest, self.name)
# try to create leading directories
elif not os.path.exists(os.path.dirname(dest)):
istools.mkdir(os.path.dirname(dest))
if os.path.isdir(dest):
raise Exception("Destination %s is a directory" % dest)
if not force:
raise Exception("File %s already exists" % dest)
# opening destination (must be local)
except Exception as e:
raise Exception("Unable to open destination file %s" % dest)
# try to open payload file
try:
f_gsrc = PipeFile(self.path, "r", progressbar=True)
f_src = gzipstream.GzipStream(stream=f_gsrc)
except Exception as e:
raise Exception("Unable to open payload file %s" % self.path)
# check if announced file size is good
if f_gsrc.size is not None and self.size != f_gsrc.size:
raise Exception("Invalid announced size on payload %s" % self.path)
shutil.copyfileobj(f_src, f_dst)
# closing fo
f_dst.close()
f_gsrc.close()
f_src.close()
# checking download size
if self.size != f_gsrc.read_size:
raise Exception("Downloading payload %s failed: Invalid size" % self.name)
if self.md5 != f_gsrc.md5:
raise Exception("Downloading payload %s failed: Invalid MD5" % self.name)
# settings file orginal rights
istools.chrights(dest, self.uid, self.gid, self.mode, self.mtime)
'''
Object representing a changelog in memory
'''
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
def __init__(self, data):
self.verbatim = ""
self.load(data)
def load(self, data):
'''
Load a changelog file
'''
version = None
lines = data.split("\n")
for line in lines:
# ignore empty lines
if len(line.strip()) == 0:
continue
# ignore comments
if line.lstrip().startswith("#"):
continue
# try to match a new version
m = re.match("\[(\d+)\]", line.lstrip())
if m is not None:
version = int(m.group(1))
self[version] = []
continue
# if line are out of a version => invalid format
if version is None:
raise Exception("Invalid format: Line outside version")
# add line to version changelog
self[version] += [line]
# save original
self.verbatim = data
def show(self, version=None, verbose=False):
'''
Show changelog for a given version or all
'''
# if no version take the hightest
if version is None:
version = max(self)
# in non verbose mode display only asked version if exists
if not verbose and version not in self:
print "chich"
return
out('#light##yellow#Changelog:#reset#')
# display asked version
out(' #yellow#Version:#reset# %s' % version)
for line in self[version]: