From ef28b0de7a72dbaf861846ed3a8e9c8dae889e68 Mon Sep 17 00:00:00 2001 From: Seblu Date: Tue, 17 May 2011 17:58:46 +0200 Subject: [PATCH] Add isrepo and update common classes --- bin/isimage | 7 +- bin/isrepo | 95 ++++++++++++++++++++++++ installsystems/__init__.py | 1 - installsystems/image.py | 139 +++++++++++++++++++++++------------ installsystems/printer.py | 14 ++-- installsystems/repository.py | 136 ++++++++++++++++++++++++++++++++++ installsystems/tarball.py | 29 ++++++++ installsystems/template.py | 2 +- 8 files changed, 368 insertions(+), 55 deletions(-) create mode 100644 bin/isrepo create mode 100644 installsystems/repository.py create mode 100644 installsystems/tarball.py diff --git a/bin/isimage b/bin/isimage index 428fc23..f0b0d87 100755 --- a/bin/isimage +++ b/bin/isimage @@ -10,6 +10,7 @@ import os import argparse import installsystems import installsystems.printer as p +import installsystems.image class DebugAction(argparse.Action): '''Set installsystems in debug mode. Argparse callback''' @@ -29,7 +30,9 @@ def init(options): p.error("%s is not writable."%parent_path) # call init from library try: - simg = installsystems.image.SourceImage(options.path, create=True) + simg = installsystems.image.SourceImage(options.path, + verbose=options.verbose, + create=True) except Exception as e: p.error("init failed: %s." % e) @@ -55,6 +58,8 @@ p_main.add_argument("-v", "--version", action="version", version=installsystems. help="show installsystems version") p_main.add_argument('-d', "--debug", action=DebugAction, nargs=0, help="active debug mode") +p_main.add_argument('-q', "--quiet", action="store_false", dest="verbose", default=True, + help="active quiet mode") subparsers = p_main.add_subparsers() # Init command parser p_init = subparsers.add_parser("init", help=init.__doc__) diff --git a/bin/isrepo b/bin/isrepo new file mode 100644 index 0000000..f558fb8 --- /dev/null +++ b/bin/isrepo @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Started 16/05/2011 by Seblu + +''' +InstallSystems Repository Manipulation Tool +''' + +import os +import argparse +import installsystems +import installsystems.repository +import installsystems.printer as p + +class DebugAction(argparse.Action): + '''Set installsystems in debug mode. Argparse callback''' + def __call__(self, parser, namespace, values, option_string=None): + installsystems.debug = True + p.debug("Debug on") + + +def init(options): + '''Create an empty fresh repo tree''' + # call init from library + try: + repo = installsystems.repository.Repository(options.image_path, + options.data_path, + verbose=options.verbose, + create=True) + except Exception as e: + p.error("init failed: %s." % e) + +def add(options): + '''Add a package to repository''' + try: + repo = installsystems.repository.Repository(options.image_path, + options.data_path, + verbose=options.verbose) + pkg = installsystems.image.PackageImage(options.path, + verbose=options.verbose) + pkg.check_md5() + repo.add(pkg) + except Exception as e: + p.error("add failed: %s." % e) + +def delete(options): + '''Remove a package from repository''' + try: + repo = installsystems.repository.Repository(options.image_path, + options.data_path, + verbose=options.verbose) + repo.delete(options.image_name, options.image_version) + except Exception as e: + p.error("del failed: %s." % e) + +# Top level argument parsing +p_main = argparse.ArgumentParser() +p_main.add_argument("-v", "--version", action="version", version=installsystems.version, + help="show installsystems version") +p_main.add_argument('-d', "--debug", action=DebugAction, nargs=0, + help="active debug mode") +p_main.add_argument('-q', "--quiet", action="store_false", dest="verbose", default=True, + help="active quiet mode") +p_main.add_argument("-I", "--image-path", dest="image_path", type=str, + help="Image repository path") +p_main.add_argument("-D", "--data-path", dest="data_path", type=str, + help="Data repository path") + +subparsers = p_main.add_subparsers() +# Init command parser +p_init = subparsers.add_parser("init", help=init.__doc__) +p_init.add_argument("image_path", nargs="?", default=argparse.SUPPRESS, + help="Path of the new image directory") +p_init.add_argument("data_path", nargs="?", default=argparse.SUPPRESS, + help="Path of the new data directory") +p_init.set_defaults(func=init) +# Add command parser +p_add = subparsers.add_parser("add", help=add.__doc__) +p_add.add_argument("path", type=str) +p_add.set_defaults(func=add) +# Del command parser +p_del = subparsers.add_parser("del", help=delete.__doc__) +p_del.add_argument("image_name", type=str) +p_del.add_argument("image_version", type=str) +p_del.set_defaults(func=delete) +# Parse and run +args = p_main.parse_args() +# Check global args +if args.image_path is None: + p_main.print_usage() + p.error("image path missing") +elif args.data_path is None: + p_main.print_usage() + p.error("data path missing") +args.func(args) diff --git a/installsystems/__init__.py b/installsystems/__init__.py index d7e6026..79ac808 100644 --- a/installsystems/__init__.py +++ b/installsystems/__init__.py @@ -11,4 +11,3 @@ version = "1-dev" debug = False import printer -import image diff --git a/installsystems/image.py b/installsystems/image.py index e3bffa1..83958b0 100644 --- a/installsystems/image.py +++ b/installsystems/image.py @@ -10,13 +10,14 @@ import os import stat import datetime import time -import tarfile import json import hashlib import StringIO import ConfigParser import subprocess +import json import installsystems.printer as p +import installsystems.tarball as tar import installsystems.template image_extension = ".img.tar.bz2" @@ -59,32 +60,32 @@ class SourceImage(Image): def create(self): '''Create an empty source image''' # create base directories - if self.verbose: p.arrow("Creating base directories") + p.arrow("Creating base directories", 1, self.verbose) try: for d in (self.base_path, self.parser_path, self.setup_path, self.data_path): os.mkdir(d) except Exception as e: raise Exception("Unable to create directory %s: %s" % (d, e)) # create example files - if self.verbose: p.arrow("Creating examples") + p.arrow("Creating examples", 1, self.verbose) try: # create description example from template - if self.verbose: p.arrow2("Creating description example") + p.arrow("Creating description example", 2, self.verbose) open(os.path.join(self.base_path, "description"), "w").write( installsystems.template.description) # create parser example from template - if self.verbose: p.arrow2("Creating parser script example") + p.arrow("Creating parser script example", 2, self.verbose) open(os.path.join(self.parser_path, "01-parser.py"), "w").write( installsystems.template.parser) # create setup example from template - if self.verbose: p.arrow2("Creating setup script example") + p.arrow("Creating setup script example", 2, self.verbose) open(os.path.join(self.setup_path, "01-setup.py"), "w").write( installsystems.template.setup) except Exception as e: raise Exception("Unable to example file: %s" % e) try: # setting rights on files in setup and parser - if self.verbose: p.arrow2("Setting executable rights on scripts") + p.arrow("Setting executable rights on scripts", 2, self.verbose) umask = os.umask(0) os.umask(umask) for path in (self.parser_path, self.setup_path): @@ -104,36 +105,35 @@ class SourceImage(Image): image_extension)) # check if free to create script tarball if os.path.exists(tarpath): - raise Exception("Tarbal already exists. Remove it before") + raise Exception("Tarball already exists. Remove it before") # printing pbzip2 status - if self.verbose: - if self.pbzip2_path: - p.arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path) - else: - p.arrow("Parallel bzip disabled") + if self.pbzip2_path: + p.arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path, 1, self.verbose) + else: + p.arrow("Parallel bzip disabled", 1, self.verbose) # Create data tarballs data_d = self.create_data_tarballs() - # generate .description.json + # generate description.json jdesc = self.generate_json_description() # creating scripts tarball - if self.verbose: p.arrow("Creating scripts tarball") - if self.verbose: p.arrow2("Name %s" % os.path.relpath(tarpath)) + p.arrow("Creating scripts tarball", 1, self.verbose) + p.arrow("Name %s" % os.path.relpath(tarpath), 2, self.verbose) try: - tarball = tarfile.open(tarpath, mode="w:bz2", dereference=True) + tarball = tar.Tarball.open(tarpath, mode="w:bz2", dereference=True) except Exception as e: raise Exception("Unable to create tarball %s: %s" % (tarpath, e)) # add .description.json - if self.verbose: p.arrow2("Add .description.json") - self.tar_add_str(tarball, tarfile.REGTYPE, 0444, ".description.json", jdesc) + p.arrow("Add .description.json", 2, self.verbose) + tarball.add_str("description.json", jdesc, tar.tarfile.REGTYPE, 0444) # add .format - if self.verbose: p.arrow2("Add .format") - self.tar_add_str(tarball, tarfile.REGTYPE, 0444, ".format", image_format) + p.arrow("Add .format", 2, self.verbose) + tarball.add_str("format", image_format, tar.tarfile.REGTYPE, 0444) # add parser scripts - if self.verbose: p.arrow2("Add parser scripts") + p.arrow("Add parser scripts", 2, self.verbose) tarball.add(self.parser_path, arcname="parser", recursive=True, filter=self.tar_scripts_filter) # add setup scripts - if self.verbose: p.arrow2("Add setup scripts") + p.arrow("Add setup scripts", 2, self.verbose) tarball.add(self.setup_path, arcname="setup", recursive=True, filter=self.tar_scripts_filter) # closing tarball file @@ -141,7 +141,7 @@ class SourceImage(Image): # compute building time t1 = time.time() dt = int(t1 - t0) - if self.verbose: p.arrow("Build time: %s" % datetime.timedelta(seconds=dt)) + p.arrow("Build time: %s" % datetime.timedelta(seconds=dt), 2, self.verbose) def data_tarballs(self): '''List all data tarballs in data directory''' @@ -156,19 +156,19 @@ class SourceImage(Image): def create_data_tarballs(self): '''Create all data tarballs in data directory''' - if self.verbose: p.arrow("Creating data tarballs") + p.arrow("Creating data tarballs", 1, self.verbose) # build list of data tarball candidate candidates = self.data_tarballs() if len(candidates) == 0: - if self.verbose: p.arrow2("No data tarball") + p.arrow("No data tarball", 2, self.verbose) return # create tarballs for candidate in candidates: path = os.path.join(self.base_path, candidate) if os.path.exists(path): - if self.verbose: p.arrow2("Tarball %s already exists." % candidate) + p.arrow("Tarball %s already exists." % candidate, 2, self.verbose) else: - if self.verbose: p.arrow2("Creating tarball %s" % candidate) + p.arrow("Creating tarball %s" % candidate, 2, self.verbose) self.create_data_tarball(path, candidates[candidate]) def create_data_tarball(self, tar_path, data_path): @@ -180,9 +180,9 @@ class SourceImage(Image): tb = open(tar_path, mode="w") p = subprocess.Popen(self.pbzip2_path, shell=False, close_fds=True, stdin=subprocess.PIPE, stdout=tb.fileno()) - tarball = tarfile.open(mode="w|", dereference=True, fileobj=p.stdin) + tarball = tar.Tarball.open(mode="w|", dereference=True, fileobj=p.stdin) else: - tarball = tarfile.open(tar_path, "w:bz2", dereference=True) + tarball = tar.Tarball.open(tar_path, "w:bz2", dereference=True) tarball.add(data_path, arcname=dname, recursive=True) # closing tarball file tarball.close() @@ -199,17 +199,6 @@ class SourceImage(Image): except Exception as e: raise Exception("Unable to create data tarball %s: %s" % (tar_path, e)) - def tar_add_str(self, tarball, ftype, mode, name, content): - '''Add a string in memory as a file in tarball''' - ti = tarfile.TarInfo(name) - ti.type = ftype - ti.mode = mode - ti.mtime = int(time.time()) - ti.uid = ti.gid = 0 - ti.uname = ti.gname = "root" - ti.size = len(content) if content is not None else 0 - tarball.addfile(ti, StringIO.StringIO(content)) - def tar_scripts_filter(self, tinfo): '''Filter files which can be included in scripts tarball''' if not tinfo.name in ("parser", "setup") and os.path.splitext(tinfo.name)[1] != ".py": @@ -221,16 +210,16 @@ class SourceImage(Image): def generate_json_description(self): '''Generate a json description file''' - if self.verbose: p.arrow("Generating JSON description") + p.arrow("Generating JSON description", 1, self.verbose) # copy description desc = self.description.copy() # timestamp image - if self.verbose: p.arrow2("Timestamping") + p.arrow("Timestamping", 2, self.verbose) desc["date"] = int(time.time()) # append data tarballs info desc["data"] = dict() for dt in self.data_tarballs(): - if self.verbose: p.arrow2("Compute MD5 of %s" % dt) + p.arrow("Compute MD5 of %s" % dt, 2, self.verbose) path = os.path.join(self.base_path, dt) desc["data"][dt] = { "size": os.path.getsize(path), "md5": self.md5_checksum(path) } @@ -241,7 +230,7 @@ class SourceImage(Image): def parse_description(self): '''Raise an exception is description file is invalid and return vars to include''' - if self.verbose: p.arrow("Parsing description") + p.arrow("Parsing description", 1, self.verbose) d = dict() try: descpath = os.path.join(self.base_path, "description") @@ -256,9 +245,65 @@ class SourceImage(Image): class PackageImage(Image): '''Packaged image manipulation class''' - def __init__(self, path): + def __init__(self, path, verbose=True): Image.__init__(self) - self.path = path + self.path = os.path.abspath(path) + self.base_path = os.path.dirname(self.path) + self.verbose = verbose + self.parse() + + def parse(self): + '''Parse tarball and extract metadata''' + # extract metadata + p.arrow("Read tarball metadata", 1, self.verbose) + try: + tarball = tar.Tarball.open(self.path) + img_format = tarball.get_str("format") + img_desc = tarball.get_str("description.json") + tarball.close() + except Exception as e: + raise e + # check format + p.arrow("Read format", 2, self.verbose) + try: + if img_format != image_format: + raise Exception() + except Exception: + raise Exception("Invalid image format") + # check description + p.arrow("Read description", 2, self.verbose) + try: + self.description = json.loads(img_desc) + except Exception as e: + raise Exception("Invalid description: %s" % e1) + + def check_md5(self): + '''Check if md5 of data tarballs are correct''' + p.arrow("Check MD5", 1, self.verbose) + databalls = self.description["data"] + for databall in databalls: + p.arrow(databall, 2, self.verbose) + md5_meta = databalls[databall]["md5"] + md5_file = self.md5_checksum(os.path.join(self.base_path, databall)) + if md5_meta != md5_file: + raise Exception("Invalid md5: %s" % databall) + + def description(self): + '''Return metadatas of a tarball''' + return self.description + + def jdescription(self): + '''Return json formated metadatas''' + return json.dumps(self.description) + + def name(self): + '''Return image name''' + return "%s-%s" % (self.description["name"], self.description["version"]) + + def databalls(self): + '''Create a dict of image and data tarballs''' + return [ os.path.join(self.base_path, d) + for d in self.description["data"] ] class DataImage(Image): '''Data image manipulation class''' diff --git a/installsystems/printer.py b/installsystems/printer.py index 53b9bfa..5898c9d 100644 --- a/installsystems/printer.py +++ b/installsystems/printer.py @@ -65,8 +65,12 @@ def debug(message, fd=sys.stderr, endl=os.linesep): if installsystems.debug: out("#light##black#%s#reset#" % message, fd, endl) -def arrow(message, fd=sys.stdout, endl=os.linesep): - out("#light##red#=>#reset# %s" % message) - -def arrow2(message, fd=sys.stdout, endl=os.linesep): - out(" #light##yellow#=>#reset# %s" % message) +def arrow(message, level, verbose, fd=sys.stdout, endl=os.linesep): + if not verbose: + return + if level == 1: + out("#light##red#=>#reset# %s" % message) + elif level == 2: + out(" #light##yellow#=>#reset# %s" % message) + elif level == 3: + out(" #light##purple#=>#reset# %s" % message) diff --git a/installsystems/repository.py b/installsystems/repository.py new file mode 100644 index 0000000..1799f2f --- /dev/null +++ b/installsystems/repository.py @@ -0,0 +1,136 @@ +# -*- python -*- +# -*- coding: utf-8 -*- +# Started 10/05/2011 by Seblu + +''' +Repository stuff +''' +import os +import time +import shutil +import json +import installsystems +import installsystems.printer as p +import installsystems.tarball as tar +import installsystems.image as image + +class Repository(object): + '''Repository class''' + + db_name = "db.tar.bz2" + last_name = "last" + repo_format = "1" + + def __init__(self, image_path, data_path, verbose=True, create=False): + '''Create an existant repository''' + self.image_path = os.path.abspath(image_path) + self.db_path = os.path.join(image_path, Repository.db_name) + self.last_path = os.path.join(image_path, Repository.last_name) + self.data_path = os.path.abspath(data_path) + self.verbose = verbose + if create: + self.create() + + def create(self): + '''Create an empty base repository''' + # create base directories + p.arrow("Creating base directories", 1, self.verbose) + try: + for d in (self.image_path, self.data_path): + if os.path.exists(d): + p.arrow("%s already exists" % os.path.relpath(d), 2, self.verbose) + else: + os.mkdir(d) + p.arrow("%s directory created" % os.path.relpath(d), 2, self.verbose) + except Exception as e: + raise Exception("Unable to create directory %s: %s" % (d, e)) + # create database + p.arrow("Creating repository database", 1, self.verbose) + if os.path.exists(self.db_path): + raise Exception("db already exists") + try: + tarball = tar.Tarball.open(self.db_path, mode="w:bz2", dereference=True) + tarball.add_str("format", Repository.repo_format, tar.tarfile.REGTYPE, 0444) + tarball.close() + except Exception as e: + raise Exception("Create database failed: %s" % e) + # create last file + p.arrow("Creating last file", 1, self.verbose) + self.update_last() + + def update_last(self): + '''Update last file to current time''' + try: + open(self.last_path, "w").write("%s\n" % int(time.time())) + except Exception as e: + raise Exception("Update last file failed: %s" % e) + + def tarballs(self, name): + '''List all tarballs (script + data)''' + ts = list() + # add script tarballs + ts.append(os.path.abspath(os.path.join(self.image_path, + "%s%s" % (name, image.image_extension)))) + db = tar.Tarball.open(self.db_path, mode='r:bz2') + jdesc = json.loads(db.get_str("%s.json" % name)) + for dt in jdesc["data"]: + ts.append(os.path.abspath(os.path.join(self.data_path, dt))) + return ts + + def add(self, package): + '''Add a packaged image to repository''' + # copy file to directory + p.arrow("Adding file to directories", 1, self.verbose) + p.arrow("Adding %s" % os.path.basename(package.path), 2, self.verbose) + shutil.copy(package.path, self.image_path) + for db in package.databalls(): + p.arrow("Adding %s" % os.path.basename(db), 2, self.verbose) + shutil.copy(db, self.data_path) + # add file to db + p.arrow("Adding metadata to db", 1, self.verbose) + name = "%s.json" % package.name() + newdb_path = "%s.new" % self.db_path + try: + db = tar.Tarball.open(self.db_path, mode='r:bz2') + newdb = tar.Tarball.open(newdb_path, mode='w:bz2') + for ti in db.getmembers(): + if ti.name != name: + newdb.addfile(ti, db.extractfile(ti)) + newdb.add_str(name, package.jdescription(), tar.tarfile.REGTYPE, 0444) + db.close() + newdb.close() + shutil.move(newdb_path, self.db_path) + except Exception as e: + raise Exception("Adding metadata fail: %s" % e) + # update last file + p.arrow("Updating last file", 1, self.verbose) + self.update_last() + + def delete(self, name, version): + '''Delete an image from repository''' + name = "%s-%s" % (name, version) + fname = "%s.json" % name + # FIXME: check tarball exists before doing this + tbs = self.tarballs(name) + # removing metadata + p.arrow("Removing metadata from db", 1, self.verbose) + newdb_path = "%s.new" % self.db_path + try: + db = tar.Tarball.open(self.db_path, mode='r:bz2') + newdb = tar.Tarball.open(newdb_path, mode='w:bz2') + for ti in db.getmembers(): + if ti.name != fname: + newdb.addfile(ti, db.extractfile(ti)) + db.close() + newdb.close() + shutil.move(newdb_path, self.db_path) + except Exception as e: + raise Exception("Removing metadata fail: %s" % e) + # removing tarballs + p.arrow("Removing tarballs", 1, self.verbose) + for tb in tbs: + p.arrow("Removing %s" % os.path.basename(tb), 2, self.verbose) + os.unlink(tb) + # update last file + p.arrow("Updating last file", 1, self.verbose) + self.update_last() diff --git a/installsystems/tarball.py b/installsystems/tarball.py new file mode 100644 index 0000000..d4b108d --- /dev/null +++ b/installsystems/tarball.py @@ -0,0 +1,29 @@ +# -*- python -*- +# -*- coding: utf-8 -*- +# Started 17/05/2011 by Seblu + +''' +Tarball wrapper +''' + +import os +import time +import tarfile +import StringIO + +class Tarball(tarfile.TarFile): + def add_str(self, name, content, ftype, mode): + '''Add a string in memory as a file in tarball''' + ti = tarfile.TarInfo(name) + ti.type = ftype + ti.mode = mode + ti.mtime = int(time.time()) + ti.uid = ti.gid = 0 + ti.uname = ti.gname = "root" + ti.size = len(content) if content is not None else 0 + self.addfile(ti, StringIO.StringIO(content)) + + def get_str(self, name): + '''Return a string from a filename in a tarball''' + ti = self.getmember(name) + return self.extractfile(ti).read() diff --git a/installsystems/template.py b/installsystems/template.py index c70c858..5a7e010 100644 --- a/installsystems/template.py +++ b/installsystems/template.py @@ -14,7 +14,7 @@ parser = """# -*- python -*- def parser(image): \t'''Method called by parser''' -\t\t pass +\t pass # vim:set ts=2 sw=2 noet: """ -- GitLab