Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
# -*- python -*-
# -*- coding: utf-8 -*-
# Started 10/05/2011 by Seblu <seblu@seblu.net>
'''
Image stuff
'''
import os
import stat
import datetime
import time
import tarfile
import json
import hashlib
import StringIO
import ConfigParser
import subprocess
import installsystems.printer as p
import installsystems.template
image_extension = ".img.tar.bz2"
image_format = "1"
class Image(object):
'''Abstract class of images'''
def __init__(self, pbzip2=True):
self.pbzip2_path = self.path_search("pbzip2") if pbzip2 else None
def path_search(self, name, path=None):
'''Search in PATH for a binary'''
path = path or os.environ["PATH"]
for d in path.split(os.pathsep):
if os.path.exists(os.path.join(d, name)):
return os.path.join(os.path.abspath(d), name)
return None
def md5_checksum(self, path):
'''Compute md5 of a file'''
m = hashlib.md5()
m.update(open(path, "r").read())
return m.hexdigest()
class SourceImage(Image):
'''Image source manipulation class'''
def __init__(self, path, verbose=True, create=False, pbzip2=True):
Image.__init__(self, pbzip2)
self.base_path = path
self.parser_path = os.path.join(path, "parser")
self.setup_path = os.path.join(path, "setup")
self.data_path = os.path.join(path, "data")
self.verbose = verbose
if create:
self.create()
self.description = self.parse_description()
def create(self):
'''Create an empty source image'''
# create base directories
if self.verbose: p.arrow("Creating base directories")
try:
for d in (self.base_path, self.parser_path, self.setup_path, self.data_path):
os.mkdir(d)
except Exception as e:
raise Exception("Unable to create directory %s: %s" % (d, e))
# create example files
if self.verbose: p.arrow("Creating examples")
try:
# create description example from template
if self.verbose: p.arrow2("Creating description example")
open(os.path.join(self.base_path, "description"), "w").write(
installsystems.template.description)
# create parser example from template
if self.verbose: p.arrow2("Creating parser script example")
open(os.path.join(self.parser_path, "01-parser.py"), "w").write(
installsystems.template.parser)
# create setup example from template
if self.verbose: p.arrow2("Creating setup script example")
open(os.path.join(self.setup_path, "01-setup.py"), "w").write(
installsystems.template.setup)
except Exception as e:
raise Exception("Unable to example file: %s" % e)
try:
# setting rights on files in setup and parser
if self.verbose: p.arrow2("Setting executable rights on scripts")
umask = os.umask(0)
os.umask(umask)
for path in (self.parser_path, self.setup_path):
for f in os.listdir(path):
pf = os.path.join(path, f)
os.chmod(pf, 0777 & ~umask)
except Exception as e:
raise Exception("Unable to set rights on %s: %s" % (pf, e))
def build(self):
'''Create packaged image'''
t0 = time.time()
# compute script tarball paths
tarpath = os.path.join(self.base_path,
"%s-%s%s" % (self.description["name"],
self.description["version"],
image_extension))
# check if free to create script tarball
if os.path.exists(tarpath):
raise Exception("Tarbal already exists. Remove it before")
# printing pbzip2 status
if self.verbose:
if self.pbzip2_path:
p.arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path)
else:
p.arrow("Parallel bzip disabled")
# Create data tarballs
data_d = self.create_data_tarballs()
# generate .description.json
jdesc = self.generate_json_description()
# creating scripts tarball
if self.verbose: p.arrow("Creating scripts tarball")
if self.verbose: p.arrow2("Name %s" % os.path.relpath(tarpath))
try:
tarball = tarfile.open(tarpath, mode="w:bz2", dereference=True)
except Exception as e:
raise Exception("Unable to create tarball %s: %s" % (tarpath, e))
# add .description.json
if self.verbose: p.arrow2("Add .description.json")
self.tar_add_str(tarball, tarfile.REGTYPE, 0444, ".description.json", jdesc)
# add .format
if self.verbose: p.arrow2("Add .format")
self.tar_add_str(tarball, tarfile.REGTYPE, 0444, ".format", image_format)
# add parser scripts
if self.verbose: p.arrow2("Add parser scripts")
tarball.add(self.parser_path, arcname="parser",
recursive=True, filter=self.tar_scripts_filter)
# add setup scripts
if self.verbose: p.arrow2("Add setup scripts")
tarball.add(self.setup_path, arcname="setup",
recursive=True, filter=self.tar_scripts_filter)
# closing tarball file
tarball.close()
# compute building time
t1 = time.time()
dt = int(t1 - t0)
if self.verbose: p.arrow("Build time: %s" % datetime.timedelta(seconds=dt))
def data_tarballs(self):
'''List all data tarballs in data directory'''
databalls = dict()
for dname in os.listdir(self.data_path):
filename = "%s-%s-%s%s" % (self.description["name"],
self.description["version"],
dname,
image_extension)
databalls[filename] = os.path.abspath(os.path.join(self.data_path, dname))
return databalls
def create_data_tarballs(self):
'''Create all data tarballs in data directory'''
if self.verbose: p.arrow("Creating data tarballs")
# build list of data tarball candidate
candidates = self.data_tarballs()
if len(candidates) == 0:
if self.verbose: p.arrow2("No data tarball")
return
# create tarballs
for candidate in candidates:
path = os.path.join(self.base_path, candidate)
if os.path.exists(path):
if self.verbose: p.arrow2("Tarball %s already exists." % candidate)
else:
if self.verbose: p.arrow2("Creating tarball %s" % candidate)
self.create_data_tarball(path, candidates[candidate])
def create_data_tarball(self, tar_path, data_path):
'''Create a data tarball'''
dname = os.path.basename(data_path)
try:
# opening file
if self.pbzip2_path:
tb = open(tar_path, mode="w")
p = subprocess.Popen(self.pbzip2_path, shell=False, close_fds=True,
stdin=subprocess.PIPE, stdout=tb.fileno())
tarball = tarfile.open(mode="w|", dereference=True, fileobj=p.stdin)
else:
tarball = tarfile.open(tar_path, "w:bz2", dereference=True)
tarball.add(data_path, arcname=dname, recursive=True)
# closing tarball file
tarball.close()
if self.pbzip2_path:
# closing pipe, needed to end pbzip2
p.stdin.close()
# waiting pbzip to terminate
r = p.wait()
# cleaning openfile
tb.close()
# status
if r != 0:
raise Exception("Data tarball %s creation return %s" % (tar_path, r))
except Exception as e:
raise Exception("Unable to create data tarball %s: %s" % (tar_path, e))
def tar_add_str(self, tarball, ftype, mode, name, content):
'''Add a string in memory as a file in tarball'''
ti = tarfile.TarInfo(name)
ti.type = ftype
ti.mode = mode
ti.mtime = int(time.time())
ti.uid = ti.gid = 0
ti.uname = ti.gname = "root"
ti.size = len(content) if content is not None else 0
tarball.addfile(ti, StringIO.StringIO(content))
def tar_scripts_filter(self, tinfo):
'''Filter files which can be included in scripts tarball'''
if not tinfo.name in ("parser", "setup") and os.path.splitext(tinfo.name)[1] != ".py":
return None
tinfo.mode = 0555
tinfo.uid = tinfo.gid = 0
tinfo.uname = tinfo.gname = "root"
return tinfo
def generate_json_description(self):
'''Generate a json description file'''
if self.verbose: p.arrow("Generating JSON description")
# copy description
desc = self.description.copy()
# timestamp image
if self.verbose: p.arrow2("Timestamping")
desc["date"] = int(time.time())
# append data tarballs info
desc["data"] = dict()
for dt in self.data_tarballs():
if self.verbose: p.arrow2("Compute MD5 of %s" % dt)
path = os.path.join(self.base_path, dt)
desc["data"][dt] = { "size": os.path.getsize(path),
"md5": self.md5_checksum(path) }
# create file
filedesc = StringIO.StringIO()
# serialize
return json.dumps(desc)
def parse_description(self):
'''Raise an exception is description file is invalid and return vars to include'''
if self.verbose: p.arrow("Parsing description")
d = dict()
try:
descpath = os.path.join(self.base_path, "description")
cp = ConfigParser.RawConfigParser()
cp.read(descpath)
for n in ("name","version", "description", "author"):
d[n] = cp.get("image", n)
except Exception as e:
raise Exception("description: %s" % e)
return d
class PackageImage(Image):
'''Packaged image manipulation class'''
def __init__(self, path):
Image.__init__(self)
self.path = path
class DataImage(Image):
'''Data image manipulation class'''
def __init__(self, path):
Image.__init__(self)
self.path = path