Newer
Older
# -*- python -*-
# -*- coding: utf-8 -*-
# Started 10/05/2011 by Seblu <seblu@seblu.net>
'''
Image stuff
'''
import os
import stat
import datetime
import time
import json
import hashlib
import StringIO
import ConfigParser
import subprocess
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
import installsystems.template
image_extension = ".img.tar.bz2"
image_format = "1"
class Image(object):
'''Abstract class of images'''
def __init__(self, pbzip2=True):
self.pbzip2_path = self.path_search("pbzip2") if pbzip2 else None
def path_search(self, name, path=None):
'''Search in PATH for a binary'''
path = path or os.environ["PATH"]
for d in path.split(os.pathsep):
if os.path.exists(os.path.join(d, name)):
return os.path.join(os.path.abspath(d), name)
return None
def md5_checksum(self, path):
'''Compute md5 of a file'''
m = hashlib.md5()
m.update(open(path, "r").read())
return m.hexdigest()
class SourceImage(Image):
'''Image source manipulation class'''
def __init__(self, path, verbose=True, create=False, pbzip2=True):
Image.__init__(self, pbzip2)
self.base_path = path
self.parser_path = os.path.join(path, "parser")
self.setup_path = os.path.join(path, "setup")
self.data_path = os.path.join(path, "data")
self.verbose = verbose
if create:
self.create()
self.description = self.parse_description()
def create(self):
'''Create an empty source image'''
# create base directories
p.arrow("Creating base directories", 1, self.verbose)
try:
for d in (self.base_path, self.parser_path, self.setup_path, self.data_path):
os.mkdir(d)
except Exception as e:
raise Exception("Unable to create directory %s: %s" % (d, e))
# create example files
p.arrow("Creating examples", 1, self.verbose)
try:
# create description example from template
p.arrow("Creating description example", 2, self.verbose)
open(os.path.join(self.base_path, "description"), "w").write(
installsystems.template.description)
# create parser example from template
p.arrow("Creating parser script example", 2, self.verbose)
open(os.path.join(self.parser_path, "01-parser.py"), "w").write(
installsystems.template.parser)
# create setup example from template
p.arrow("Creating setup script example", 2, self.verbose)
open(os.path.join(self.setup_path, "01-setup.py"), "w").write(
installsystems.template.setup)
except Exception as e:
raise Exception("Unable to example file: %s" % e)
try:
# setting rights on files in setup and parser
p.arrow("Setting executable rights on scripts", 2, self.verbose)
umask = os.umask(0)
os.umask(umask)
for path in (self.parser_path, self.setup_path):
for f in os.listdir(path):
pf = os.path.join(path, f)
os.chmod(pf, 0777 & ~umask)
except Exception as e:
raise Exception("Unable to set rights on %s: %s" % (pf, e))
def build(self):
'''Create packaged image'''
t0 = time.time()
# compute script tarball paths
tarpath = os.path.join(self.base_path,
"%s-%s%s" % (self.description["name"],
self.description["version"],
image_extension))
# check if free to create script tarball
if os.path.exists(tarpath):
raise Exception("Tarball already exists. Remove it before")
if self.pbzip2_path:
p.arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path, 1, self.verbose)
else:
p.arrow("Parallel bzip disabled", 1, self.verbose)
# Create data tarballs
data_d = self.create_data_tarballs()
jdesc = self.generate_json_description()
# creating scripts tarball
p.arrow("Creating scripts tarball", 1, self.verbose)
p.arrow("Name %s" % os.path.relpath(tarpath), 2, self.verbose)
tarball = tar.Tarball.open(tarpath, mode="w:bz2", dereference=True)
except Exception as e:
raise Exception("Unable to create tarball %s: %s" % (tarpath, e))
# add .description.json
p.arrow("Add .description.json", 2, self.verbose)
tarball.add_str("description.json", jdesc, tar.tarfile.REGTYPE, 0444)
p.arrow("Add .format", 2, self.verbose)
tarball.add_str("format", image_format, tar.tarfile.REGTYPE, 0444)
p.arrow("Add parser scripts", 2, self.verbose)
tarball.add(self.parser_path, arcname="parser",
recursive=True, filter=self.tar_scripts_filter)
# add setup scripts
p.arrow("Add setup scripts", 2, self.verbose)
tarball.add(self.setup_path, arcname="setup",
recursive=True, filter=self.tar_scripts_filter)
# closing tarball file
tarball.close()
# compute building time
t1 = time.time()
dt = int(t1 - t0)
p.arrow("Build time: %s" % datetime.timedelta(seconds=dt), 2, self.verbose)
def data_tarballs(self):
'''List all data tarballs in data directory'''
databalls = dict()
for dname in os.listdir(self.data_path):
filename = "%s-%s-%s%s" % (self.description["name"],
self.description["version"],
dname,
image_extension)
databalls[filename] = os.path.abspath(os.path.join(self.data_path, dname))
return databalls
def create_data_tarballs(self):
'''Create all data tarballs in data directory'''
p.arrow("Creating data tarballs", 1, self.verbose)
# build list of data tarball candidate
candidates = self.data_tarballs()
if len(candidates) == 0:
p.arrow("No data tarball", 2, self.verbose)
return
# create tarballs
for candidate in candidates:
path = os.path.join(self.base_path, candidate)
if os.path.exists(path):
p.arrow("Tarball %s already exists." % candidate, 2, self.verbose)
p.arrow("Creating tarball %s" % candidate, 2, self.verbose)
self.create_data_tarball(path, candidates[candidate])
def create_data_tarball(self, tar_path, data_path):
'''Create a data tarball'''
dname = os.path.basename(data_path)
try:
# opening file
if self.pbzip2_path:
tb = open(tar_path, mode="w")
p = subprocess.Popen(self.pbzip2_path, shell=False, close_fds=True,
stdin=subprocess.PIPE, stdout=tb.fileno())
tarball = tar.Tarball.open(mode="w|", dereference=True, fileobj=p.stdin)
tarball = tar.Tarball.open(tar_path, "w:bz2", dereference=True)
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
tarball.add(data_path, arcname=dname, recursive=True)
# closing tarball file
tarball.close()
if self.pbzip2_path:
# closing pipe, needed to end pbzip2
p.stdin.close()
# waiting pbzip to terminate
r = p.wait()
# cleaning openfile
tb.close()
# status
if r != 0:
raise Exception("Data tarball %s creation return %s" % (tar_path, r))
except Exception as e:
raise Exception("Unable to create data tarball %s: %s" % (tar_path, e))
def tar_scripts_filter(self, tinfo):
'''Filter files which can be included in scripts tarball'''
if not tinfo.name in ("parser", "setup") and os.path.splitext(tinfo.name)[1] != ".py":
return None
tinfo.mode = 0555
tinfo.uid = tinfo.gid = 0
tinfo.uname = tinfo.gname = "root"
return tinfo
def generate_json_description(self):
'''Generate a json description file'''
p.arrow("Generating JSON description", 1, self.verbose)
# copy description
desc = self.description.copy()
# timestamp image
desc["date"] = int(time.time())
# append data tarballs info
desc["data"] = dict()
for dt in self.data_tarballs():
p.arrow("Compute MD5 of %s" % dt, 2, self.verbose)
path = os.path.join(self.base_path, dt)
desc["data"][dt] = { "size": os.path.getsize(path),
"md5": self.md5_checksum(path) }
# create file
filedesc = StringIO.StringIO()
# serialize
return json.dumps(desc)
def parse_description(self):
'''Raise an exception is description file is invalid and return vars to include'''
p.arrow("Parsing description", 1, self.verbose)
d = dict()
try:
descpath = os.path.join(self.base_path, "description")
cp = ConfigParser.RawConfigParser()
cp.read(descpath)
for n in ("name","version", "description", "author"):
d[n] = cp.get("image", n)
except Exception as e:
raise Exception("description: %s" % e)
return d
class PackageImage(Image):
'''Packaged image manipulation class'''
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
self.path = os.path.abspath(path)
self.base_path = os.path.dirname(self.path)
self.verbose = verbose
self.parse()
def parse(self):
'''Parse tarball and extract metadata'''
# extract metadata
p.arrow("Read tarball metadata", 1, self.verbose)
try:
tarball = tar.Tarball.open(self.path)
img_format = tarball.get_str("format")
img_desc = tarball.get_str("description.json")
tarball.close()
except Exception as e:
raise e
# check format
p.arrow("Read format", 2, self.verbose)
try:
if img_format != image_format:
raise Exception()
except Exception:
raise Exception("Invalid image format")
# check description
p.arrow("Read description", 2, self.verbose)
try:
self.description = json.loads(img_desc)
except Exception as e:
raise Exception("Invalid description: %s" % e1)
def check_md5(self):
'''Check if md5 of data tarballs are correct'''
p.arrow("Check MD5", 1, self.verbose)
databalls = self.description["data"]
for databall in databalls:
p.arrow(databall, 2, self.verbose)
md5_meta = databalls[databall]["md5"]
md5_file = self.md5_checksum(os.path.join(self.base_path, databall))
if md5_meta != md5_file:
raise Exception("Invalid md5: %s" % databall)
def description(self):
'''Return metadatas of a tarball'''
return self.description
def jdescription(self):
'''Return json formated metadatas'''
return json.dumps(self.description)
def name(self):
'''Return image name'''
return "%s-%s" % (self.description["name"], self.description["version"])
def databalls(self):
'''Create a dict of image and data tarballs'''
return [ os.path.join(self.base_path, d)
for d in self.description["data"] ]