diff --git a/bin/isimage b/bin/isimage index f0b0d8701e7f0d27b253c992c16ff05cead98210..f9bc865b51b7a2e7412e66be3f6756b57d02c289 100755 --- a/bin/isimage +++ b/bin/isimage @@ -9,48 +9,47 @@ InstallSystems Image Manipulation Tool import os import argparse import installsystems -import installsystems.printer as p -import installsystems.image +from installsystems.printer import * +from installsystems.image import SourceImage class DebugAction(argparse.Action): '''Set installsystems in debug mode. Argparse callback''' def __call__(self, parser, namespace, values, option_string=None): - installsystems.debug = True - p.debug("Debug on") + if installsystems.debug == False: + installsystems.debug = True + debug("Debug on") def init(options): '''Create an empty fresh source image tree''' # Directory must not exists if os.path.exists(options.path) and os.path.isdir(options.path): - p.error("Directory already exists: %s" % options.path) + error("Directory already exists: %s" % options.path) # Check if parent path is writable parent_path = os.path.abspath(os.path.join(options.path, "../")) if not os.access(parent_path, os.W_OK): - p.error("%s is not writable."%parent_path) + error("%s is not writable."%parent_path) # call init from library try: - simg = installsystems.image.SourceImage(options.path, - verbose=options.verbose, - create=True) + simg = SourceImage.create(options.path, options.verbose) except Exception as e: - p.error("init failed: %s." % e) + error("init failed: %s." % e) def build(options): '''Create a package image''' for d in ("", "parser", "setup", "data"): rp = os.path.join(options.path, d) if not os.path.exists(rp): - p.error("Missing directory: %s" % rp) + error("Missing directory: %s" % rp) if not os.path.isdir(rp): - p.error("Not a directory: %s" % rp) + error("Not a directory: %s" % rp) if not os.access(rp, os.R_OK|os.X_OK): - p.error("Unable to access to %s" % rp) + error("Unable to access to %s" % rp) try: - simg = installsystems.image.SourceImage(options.path) + simg = SourceImage(options.path) simg.build() except Exception as e: - p.error("build failed: %s." % e) + error("build failed: %s." % e) # Top level argument parsing p_main = argparse.ArgumentParser() diff --git a/bin/isinstall b/bin/isinstall new file mode 100755 index 0000000000000000000000000000000000000000..e462a391e2378e2c0ca462099305137abdf52dc3 --- /dev/null +++ b/bin/isinstall @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Started 18/05/2011 by Seblu + +''' +InstallSystems Installation Tool +''' + +import os +import argparse +import installsystems +from installsystems.printer import * +from installsystems.repository import RepositoryCache + +class DebugAction(argparse.Action): + '''Set installsystems in debug mode. Argparse callback''' + def __call__(self, parser, namespace, values, option_string=None): + if installsystems.debug == False: + installsystems.debug = True + debug("Debug on") + + +# Top level argument parsing +p_main = argparse.ArgumentParser() +p_main.add_argument("-v", "--version", action="version", version=installsystems.version, + help="show installsystems version") +p_main.add_argument('-d', "--debug", action=DebugAction, nargs=0, + help="active debug mode") +p_main.add_argument('-q', "--quiet", action="store_false", dest="verbose", default=True, + help="active quiet mode") +p_main.add_argument("-I", "--image-path", dest="image_path", type=str, default=None, + help="remote image repository path") +p_main.add_argument("-D", "--data-path", dest="data_path", type=str, default=None, + help="remote data repository path") +p_main.add_argument("-C", "--cache-path", dest="cache_path", type=str, + default="/var/cache/isinstall", + help="local cache repository path") +p_main.add_argument("-V", "--image-version", dest="image_version", type=int, default=None, + help="specific image version") +p_main.add_argument("image_name", type=str, + help="image to install") + +# program entry point +try: + # Partial parse + args = p_main.parse_known_args()[0] + # create repo cache object + repocache = RepositoryCache(args.cache_path, verbose=args.verbose) + # register command ligne repo + if args.image_path is not None: + repocache.register("cmdline", args.image_path, args.data_path) + # update remote info if available + repocache.update() + # get image package + pkg = repocache.get(args.image_name, args.image_version) + # create global dict, used to share and transmit data between scripts + # and add parser object to allow parser script to extend parser + gl = { "parser": p_main } + # run parser scripts + pkg.run_parser(gl) + # call parser again, with extended attributes + args = p_main.parse_args() + # remove parser object from global + del gl["parser"] + gl["args"] = args + # run setup scripts + pkg.run_setup(gl) +except Exception as e: + error(e) diff --git a/bin/isrepo b/bin/isrepo old mode 100644 new mode 100755 index f558fb8037af20c15b1fc4dd2091c627c30277ad..23b59ac048f73dd2c007e085c8761e009f057e68 --- a/bin/isrepo +++ b/bin/isrepo @@ -9,49 +9,44 @@ InstallSystems Repository Manipulation Tool import os import argparse import installsystems -import installsystems.repository -import installsystems.printer as p +from installsystems.printer import * +from installsystems.repository import Repository +from installsystems.image import PackageImage + class DebugAction(argparse.Action): '''Set installsystems in debug mode. Argparse callback''' def __call__(self, parser, namespace, values, option_string=None): - installsystems.debug = True - p.debug("Debug on") + if installsystems.debug == False: + installsystems.debug = True + debug("Debug on") def init(options): '''Create an empty fresh repo tree''' # call init from library try: - repo = installsystems.repository.Repository(options.image_path, - options.data_path, - verbose=options.verbose, - create=True) + Repository.create(options.image_path, options.data_path, options.verbose) except Exception as e: - p.error("init failed: %s." % e) + error("init failed: %s." % e) def add(options): '''Add a package to repository''' try: - repo = installsystems.repository.Repository(options.image_path, - options.data_path, - verbose=options.verbose) - pkg = installsystems.image.PackageImage(options.path, - verbose=options.verbose) + repo = Repository(options.image_path, options.data_path, options.verbose) + pkg = PackageImage(options.path, options.verbose) pkg.check_md5() repo.add(pkg) except Exception as e: - p.error("add failed: %s." % e) + error("add failed: %s." % e) def delete(options): '''Remove a package from repository''' try: - repo = installsystems.repository.Repository(options.image_path, - options.data_path, - verbose=options.verbose) + repo = Repository(options.image_path, options.data_path, options.verbose) repo.delete(options.image_name, options.image_version) except Exception as e: - p.error("del failed: %s." % e) + error("del failed: %s." % e) # Top level argument parsing p_main = argparse.ArgumentParser() @@ -62,9 +57,9 @@ p_main.add_argument('-d', "--debug", action=DebugAction, nargs=0, p_main.add_argument('-q', "--quiet", action="store_false", dest="verbose", default=True, help="active quiet mode") p_main.add_argument("-I", "--image-path", dest="image_path", type=str, - help="Image repository path") + help="image repository path") p_main.add_argument("-D", "--data-path", dest="data_path", type=str, - help="Data repository path") + help="data repository path") subparsers = p_main.add_subparsers() # Init command parser @@ -88,8 +83,8 @@ args = p_main.parse_args() # Check global args if args.image_path is None: p_main.print_usage() - p.error("image path missing") + error("image path missing") elif args.data_path is None: p_main.print_usage() - p.error("data path missing") + error("data path missing") args.func(args) diff --git a/installsystems/__init__.py b/installsystems/__init__.py index 79ac8080e5555d97dffde0c67c6bdec14917c4d6..4f5d9eab0741fe4ab74141fda0686cb9ae246bf9 100644 --- a/installsystems/__init__.py +++ b/installsystems/__init__.py @@ -10,4 +10,4 @@ canonical_name="installsystems" version = "1-dev" debug = False -import printer +__all__ = [] diff --git a/installsystems/database.py b/installsystems/database.py new file mode 100644 index 0000000000000000000000000000000000000000..604330843c7a76418732dec4415a0d473a971fe4 --- /dev/null +++ b/installsystems/database.py @@ -0,0 +1,91 @@ +# -*- python -*- +# -*- coding: utf-8 -*- +# Started 24/05/2011 by Seblu + +''' +Database stuff +''' + +import json +import os +import shutil +import tarfile +from installsystems.tarball import Tarball +from installsystems.printer import * + +class Database(object): + '''Abstract repo database stuff''' + + db_format = "1" + + @classmethod + def create(cls, path, verbose=True): + arrow("Creating repository database", 1, verbose) + dbpath = os.path.abspath(path) + if os.path.exists(dbpath): + raise Exception("db already exists") + try: + tarball = Tarball.open(dbpath, mode="w:bz2", dereference=True) + tarball.add_str("format", Database.db_format, tarfile.REGTYPE, 0444) + tarball.close() + except Exception as e: + raise Exception("Create database failed: %s" % e) + return cls(path, verbose) + + def __init__(self, path, verbose=True): + self.path = os.path.abspath(path) + self.verbose = verbose + + def add(self, package): + '''Add a packaged image to a db''' + arrow("Adding metadata to db", 1, self.verbose) + name = "%s.json" % package.name() + newdb_path = "%s.new" % self.path + try: + db = Tarball.open(self.path, mode='r:bz2') + newdb = Tarball.open(newdb_path, mode='w:bz2') + for ti in db.getmembers(): + if ti.name != name: + newdb.addfile(ti, db.extractfile(ti)) + newdb.add_str(name, package.jdescription(), tarfile.REGTYPE, 0444) + db.close() + newdb.close() + shutil.move(newdb_path, self.path) + except Exception as e: + raise Exception("Adding metadata fail: %s" % e) + + def delete(self, name, version): + '''Deltete a packaged image''' + arrow("Removing metadata from db", 1, self.verbose) + newdb_path = "%s.new" % self.path + try: + db = Tarball.open(self.path, mode='r:bz2') + newdb = Tarball.open(newdb_path, mode='w:bz2') + for ti in db.getmembers(): + if ti.name != fname: + newdb.addfile(ti, db.extractfile(ti)) + db.close() + newdb.close() + shutil.move(newdb_path, self.path) + except Exception as e: + raise Exception("Removing metadata fail: %s" % e) + + def find(self, name, version=None): + '''Find last version of an image''' + try: + tarb = Tarball.open(self.path, mode='r:bz2') + candidates = [ int((os.path.splitext(tpname)[0]).rsplit("-", 1)[1]) + for tpname in tarb.getnames("%s-\d+.json" % name) ] + tarb.close() + except Exception as e: + raise Exception("Find in db %s fail: %s" % (self.path, e)) + # no candidates => go west + if len(candidates) == 0: + return None + # get last version + if version is None: + version = max(candidates) + # check if version exists + if int(version) not in candidates: + return None + return (name, version) diff --git a/installsystems/image.py b/installsystems/image.py index 83958b05dfc9e7144498c88a5dd5bb3667786e9d..e8d47dc01140bda3c1bff6a0252bfef1e8eac75f 100644 --- a/installsystems/image.py +++ b/installsystems/image.py @@ -16,16 +16,19 @@ import StringIO import ConfigParser import subprocess import json -import installsystems.printer as p -import installsystems.tarball as tar +import tarfile import installsystems.template +from installsystems.printer import * +from installsystems.tarball import Tarball -image_extension = ".img.tar.bz2" -image_format = "1" class Image(object): '''Abstract class of images''' + image_extension = ".isimage" + image_payload = ".isdata" + image_format = "1" + def __init__(self, pbzip2=True): self.pbzip2_path = self.path_search("pbzip2") if pbzip2 else None @@ -46,54 +49,57 @@ class Image(object): class SourceImage(Image): '''Image source manipulation class''' - def __init__(self, path, verbose=True, create=False, pbzip2=True): + def __init__(self, path, verbose=True, pbzip2=True): Image.__init__(self, pbzip2) self.base_path = path self.parser_path = os.path.join(path, "parser") self.setup_path = os.path.join(path, "setup") self.data_path = os.path.join(path, "data") self.verbose = verbose - if create: - self.create() self.description = self.parse_description() - def create(self): + @classmethod + def create(cls, path, verbose=True, pbzip2=True): '''Create an empty source image''' + parser_path = os.path.join(path, "parser") + setup_path = os.path.join(path, "setup") + data_path = os.path.join(path, "data") # create base directories - p.arrow("Creating base directories", 1, self.verbose) + arrow("Creating base directories", 1, verbose) try: - for d in (self.base_path, self.parser_path, self.setup_path, self.data_path): + for d in (path, parser_path, setup_path, data_path): os.mkdir(d) except Exception as e: - raise Exception("Unable to create directory %s: %s" % (d, e)) + raise Exception("Unable to create directory: %s: %s" % (d, e)) # create example files - p.arrow("Creating examples", 1, self.verbose) + arrow("Creating examples", 1, verbose) try: # create description example from template - p.arrow("Creating description example", 2, self.verbose) - open(os.path.join(self.base_path, "description"), "w").write( + arrow("Creating description example", 2, verbose) + open(os.path.join(path, "description"), "w").write( installsystems.template.description) # create parser example from template - p.arrow("Creating parser script example", 2, self.verbose) - open(os.path.join(self.parser_path, "01-parser.py"), "w").write( + arrow("Creating parser script example", 2, verbose) + open(os.path.join(parser_path, "01-parser.py"), "w").write( installsystems.template.parser) # create setup example from template - p.arrow("Creating setup script example", 2, self.verbose) - open(os.path.join(self.setup_path, "01-setup.py"), "w").write( + arrow("Creating setup script example", 2, verbose) + open(os.path.join(setup_path, "01-setup.py"), "w").write( installsystems.template.setup) except Exception as e: raise Exception("Unable to example file: %s" % e) try: # setting rights on files in setup and parser - p.arrow("Setting executable rights on scripts", 2, self.verbose) + arrow("Setting executable rights on scripts", 2, verbose) umask = os.umask(0) os.umask(umask) - for path in (self.parser_path, self.setup_path): - for f in os.listdir(path): - pf = os.path.join(path, f) + for dpath in (parser_path, setup_path): + for f in os.listdir(dpath): + pf = os.path.join(dpath, f) os.chmod(pf, 0777 & ~umask) except Exception as e: raise Exception("Unable to set rights on %s: %s" % (pf, e)) + return cls(path, verbose, pbzip2) def build(self): '''Create packaged image''' @@ -102,38 +108,38 @@ class SourceImage(Image): tarpath = os.path.join(self.base_path, "%s-%s%s" % (self.description["name"], self.description["version"], - image_extension)) + self.image_extension)) # check if free to create script tarball if os.path.exists(tarpath): raise Exception("Tarball already exists. Remove it before") # printing pbzip2 status if self.pbzip2_path: - p.arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path, 1, self.verbose) + arrow("Parallel bzip2 enabled (%s)" % self.pbzip2_path, 1, self.verbose) else: - p.arrow("Parallel bzip disabled", 1, self.verbose) + arrow("Parallel bzip disabled", 1, self.verbose) # Create data tarballs data_d = self.create_data_tarballs() # generate description.json jdesc = self.generate_json_description() # creating scripts tarball - p.arrow("Creating scripts tarball", 1, self.verbose) - p.arrow("Name %s" % os.path.relpath(tarpath), 2, self.verbose) + arrow("Creating scripts tarball", 1, self.verbose) + arrow("Name %s" % os.path.relpath(tarpath), 2, self.verbose) try: - tarball = tar.Tarball.open(tarpath, mode="w:bz2", dereference=True) + tarball = Tarball.open(tarpath, mode="w:bz2", dereference=True) except Exception as e: raise Exception("Unable to create tarball %s: %s" % (tarpath, e)) # add .description.json - p.arrow("Add .description.json", 2, self.verbose) - tarball.add_str("description.json", jdesc, tar.tarfile.REGTYPE, 0444) + arrow("Add .description.json", 2, self.verbose) + tarball.add_str("description.json", jdesc, tarfile.REGTYPE, 0444) # add .format - p.arrow("Add .format", 2, self.verbose) - tarball.add_str("format", image_format, tar.tarfile.REGTYPE, 0444) + arrow("Add .format", 2, self.verbose) + tarball.add_str("format", self.image_format, tarfile.REGTYPE, 0444) # add parser scripts - p.arrow("Add parser scripts", 2, self.verbose) + arrow("Add parser scripts", 2, self.verbose) tarball.add(self.parser_path, arcname="parser", recursive=True, filter=self.tar_scripts_filter) # add setup scripts - p.arrow("Add setup scripts", 2, self.verbose) + arrow("Add setup scripts", 2, self.verbose) tarball.add(self.setup_path, arcname="setup", recursive=True, filter=self.tar_scripts_filter) # closing tarball file @@ -141,7 +147,7 @@ class SourceImage(Image): # compute building time t1 = time.time() dt = int(t1 - t0) - p.arrow("Build time: %s" % datetime.timedelta(seconds=dt), 2, self.verbose) + arrow("Build time: %s" % datetime.timedelta(seconds=dt), 2, self.verbose) def data_tarballs(self): '''List all data tarballs in data directory''' @@ -150,25 +156,25 @@ class SourceImage(Image): filename = "%s-%s-%s%s" % (self.description["name"], self.description["version"], dname, - image_extension) + self.image_payload) databalls[filename] = os.path.abspath(os.path.join(self.data_path, dname)) return databalls def create_data_tarballs(self): '''Create all data tarballs in data directory''' - p.arrow("Creating data tarballs", 1, self.verbose) + arrow("Creating data tarballs", 1, self.verbose) # build list of data tarball candidate candidates = self.data_tarballs() if len(candidates) == 0: - p.arrow("No data tarball", 2, self.verbose) + arrow("No data tarball", 2, self.verbose) return # create tarballs for candidate in candidates: path = os.path.join(self.base_path, candidate) if os.path.exists(path): - p.arrow("Tarball %s already exists." % candidate, 2, self.verbose) + arrow("Tarball %s already exists." % candidate, 2, self.verbose) else: - p.arrow("Creating tarball %s" % candidate, 2, self.verbose) + arrow("Creating tarball %s" % candidate, 2, self.verbose) self.create_data_tarball(path, candidates[candidate]) def create_data_tarball(self, tar_path, data_path): @@ -180,9 +186,9 @@ class SourceImage(Image): tb = open(tar_path, mode="w") p = subprocess.Popen(self.pbzip2_path, shell=False, close_fds=True, stdin=subprocess.PIPE, stdout=tb.fileno()) - tarball = tar.Tarball.open(mode="w|", dereference=True, fileobj=p.stdin) + tarball = Tarball.open(mode="w|", dereference=True, fileobj=p.stdin) else: - tarball = tar.Tarball.open(tar_path, "w:bz2", dereference=True) + tarball = Tarball.open(tar_path, "w:bz2", dereference=True) tarball.add(data_path, arcname=dname, recursive=True) # closing tarball file tarball.close() @@ -210,16 +216,16 @@ class SourceImage(Image): def generate_json_description(self): '''Generate a json description file''' - p.arrow("Generating JSON description", 1, self.verbose) + arrow("Generating JSON description", 1, self.verbose) # copy description desc = self.description.copy() # timestamp image - p.arrow("Timestamping", 2, self.verbose) + arrow("Timestamping", 2, self.verbose) desc["date"] = int(time.time()) # append data tarballs info desc["data"] = dict() for dt in self.data_tarballs(): - p.arrow("Compute MD5 of %s" % dt, 2, self.verbose) + arrow("Compute MD5 of %s" % dt, 2, self.verbose) path = os.path.join(self.base_path, dt) desc["data"][dt] = { "size": os.path.getsize(path), "md5": self.md5_checksum(path) } @@ -230,7 +236,7 @@ class SourceImage(Image): def parse_description(self): '''Raise an exception is description file is invalid and return vars to include''' - p.arrow("Parsing description", 1, self.verbose) + arrow("Parsing description", 1, self.verbose) d = dict() try: descpath = os.path.join(self.base_path, "description") @@ -250,28 +256,21 @@ class PackageImage(Image): self.path = os.path.abspath(path) self.base_path = os.path.dirname(self.path) self.verbose = verbose + self.tarball = Tarball.open(self.path, mode='r:bz2') self.parse() def parse(self): '''Parse tarball and extract metadata''' # extract metadata - p.arrow("Read tarball metadata", 1, self.verbose) - try: - tarball = tar.Tarball.open(self.path) - img_format = tarball.get_str("format") - img_desc = tarball.get_str("description.json") - tarball.close() - except Exception as e: - raise e + arrow("Read tarball metadata", 1, self.verbose) + img_format = self.tarball.get_str("format") + img_desc = self.tarball.get_str("description.json") # check format - p.arrow("Read format", 2, self.verbose) - try: - if img_format != image_format: - raise Exception() - except Exception: - raise Exception("Invalid image format") + arrow("Read format", 2, self.verbose) + if img_format != self.image_format: + raise Exception("Invalid tarball image format") # check description - p.arrow("Read description", 2, self.verbose) + arrow("Read description", 2, self.verbose) try: self.description = json.loads(img_desc) except Exception as e: @@ -279,10 +278,10 @@ class PackageImage(Image): def check_md5(self): '''Check if md5 of data tarballs are correct''' - p.arrow("Check MD5", 1, self.verbose) + arrow("Check MD5", 1, self.verbose) databalls = self.description["data"] for databall in databalls: - p.arrow(databall, 2, self.verbose) + arrow(databall, 2, self.verbose) md5_meta = databalls[databall]["md5"] md5_file = self.md5_checksum(os.path.join(self.base_path, databall)) if md5_meta != md5_file: @@ -305,9 +304,23 @@ class PackageImage(Image): return [ os.path.join(self.base_path, d) for d in self.description["data"] ] -class DataImage(Image): - '''Data image manipulation class''' + def run_parser(self, gl): + '''Run parser scripts''' + self.run_scripts(gl, "parser") - def __init__(self, path): - Image.__init__(self) - self.path = path + def run_setup(self, gl): + '''Run setup scripts''' + self.run_scripts(gl, "setup") + + def run_scripts(self, gl, directory): + '''Run scripts in a tarball directory''' + arrow("Run %s" % directory, 1, self.verbose) + # get list of parser scripts + l_scripts = self.tarball.getnames("%s/.*\.py" % directory) + # order matter! + l_scripts.sort() + # run scripts + for n_scripts in l_scripts: + arrow(os.path.basename(n_scripts), 2, self.verbose) + s_scripts = self.tarball.get_str(n_scripts) + exec(s_scripts, gl, dict()) diff --git a/installsystems/repository.py b/installsystems/repository.py index 1799f2f3ddf4113091137c9bd624022ac6a5aa13..dfc71a7c59be5aace789f00696662eebce55f3fb 100644 --- a/installsystems/repository.py +++ b/installsystems/repository.py @@ -5,57 +5,75 @@ ''' Repository stuff ''' + import os import time import shutil import json import installsystems -import installsystems.printer as p -import installsystems.tarball as tar -import installsystems.image as image +from installsystems.printer import * +from installsystems.tarball import Tarball +from installsystems.image import Image, PackageImage +from installsystems.database import Database + +class RepositoryBase(object): + '''Base repository class''' + + def complete_path(self, path): + '''Format a path to be complete''' + if self.path_type(path) in ('http', 'ssh'): + return path + else: + return os.path.abspath(path) + + def path_type(self, path): + '''Return path type''' + if path.startswith("http://") or path.startswith("https://"): + return "http" + # elif path.startswith("ssh://"): + # return "ssh" + return "file" + + def cp(self, source, destination): + '''Copy a source to destination. Take care of path type''' + stype = self.path_type(source) + dtype = self.path_type(destination) + if stype == dtype == "file": + shutil.copy(source, destination) + elif stype == "file" and dtype == "": + pass -class Repository(object): +class Repository(RepositoryBase): '''Repository class''' - db_name = "db.tar.bz2" last_name = "last" - repo_format = "1" - def __init__(self, image_path, data_path, verbose=True, create=False): - '''Create an existant repository''' + def __init__(self, image_path, data_path, verbose=True): self.image_path = os.path.abspath(image_path) - self.db_path = os.path.join(image_path, Repository.db_name) - self.last_path = os.path.join(image_path, Repository.last_name) + self.last_path = os.path.join(image_path, self.last_name) self.data_path = os.path.abspath(data_path) self.verbose = verbose - if create: - self.create() + self.db = Database(os.path.join(image_path, "db"), verbose=self.verbose) - def create(self): + @classmethod + def create(cls, image_path, data_path, verbose=True): '''Create an empty base repository''' # create base directories - p.arrow("Creating base directories", 1, self.verbose) + arrow("Creating base directories", 1, verbose) try: - for d in (self.image_path, self.data_path): + for d in (image_path, data_path): if os.path.exists(d): - p.arrow("%s already exists" % os.path.relpath(d), 2, self.verbose) + arrow("%s already exists" % os.path.relpath(d), 2, verbose) else: os.mkdir(d) - p.arrow("%s directory created" % os.path.relpath(d), 2, self.verbose) + arrow("%s directory created" % os.path.relpath(d), 2, verbose) except Exception as e: raise Exception("Unable to create directory %s: %s" % (d, e)) # create database - p.arrow("Creating repository database", 1, self.verbose) - if os.path.exists(self.db_path): - raise Exception("db already exists") - try: - tarball = tar.Tarball.open(self.db_path, mode="w:bz2", dereference=True) - tarball.add_str("format", Repository.repo_format, tar.tarfile.REGTYPE, 0444) - tarball.close() - except Exception as e: - raise Exception("Create database failed: %s" % e) + d = Database.create(os.path.join(image_path, "db"), verbose=verbose) # create last file - p.arrow("Creating last file", 1, self.verbose) + arrow("Creating last file", 1, verbose) + self = cls(image_path, data_path, verbose) self.update_last() def update_last(self): @@ -65,45 +83,27 @@ class Repository(object): except Exception as e: raise Exception("Update last file failed: %s" % e) - def tarballs(self, name): - '''List all tarballs (script + data)''' - ts = list() - # add script tarballs - ts.append(os.path.abspath(os.path.join(self.image_path, - "%s%s" % (name, image.image_extension)))) - db = tar.Tarball.open(self.db_path, mode='r:bz2') - jdesc = json.loads(db.get_str("%s.json" % name)) - for dt in jdesc["data"]: - ts.append(os.path.abspath(os.path.join(self.data_path, dt))) - return ts + def last(self): + '''Return the last value''' + try: + return int(open(self.last_path, "r").read().rstrip()) + except Exception as e: + raise Exception("Read last file failed: %s" % e) + return 0 def add(self, package): '''Add a packaged image to repository''' # copy file to directory - p.arrow("Adding file to directories", 1, self.verbose) - p.arrow("Adding %s" % os.path.basename(package.path), 2, self.verbose) + arrow("Adding file to directories", 1, self.verbose) + arrow("Adding %s" % os.path.basename(package.path), 2, self.verbose) shutil.copy(package.path, self.image_path) for db in package.databalls(): - p.arrow("Adding %s" % os.path.basename(db), 2, self.verbose) + arrow("Adding %s" % os.path.basename(db), 2, self.verbose) shutil.copy(db, self.data_path) # add file to db - p.arrow("Adding metadata to db", 1, self.verbose) - name = "%s.json" % package.name() - newdb_path = "%s.new" % self.db_path - try: - db = tar.Tarball.open(self.db_path, mode='r:bz2') - newdb = tar.Tarball.open(newdb_path, mode='w:bz2') - for ti in db.getmembers(): - if ti.name != name: - newdb.addfile(ti, db.extractfile(ti)) - newdb.add_str(name, package.jdescription(), tar.tarfile.REGTYPE, 0444) - db.close() - newdb.close() - shutil.move(newdb_path, self.db_path) - except Exception as e: - raise Exception("Adding metadata fail: %s" % e) + self.db.add(package) # update last file - p.arrow("Updating last file", 1, self.verbose) + arrow("Updating last file", 1, self.verbose) self.update_last() def delete(self, name, version): @@ -113,24 +113,114 @@ class Repository(object): # FIXME: check tarball exists before doing this tbs = self.tarballs(name) # removing metadata - p.arrow("Removing metadata from db", 1, self.verbose) - newdb_path = "%s.new" % self.db_path - try: - db = tar.Tarball.open(self.db_path, mode='r:bz2') - newdb = tar.Tarball.open(newdb_path, mode='w:bz2') - for ti in db.getmembers(): - if ti.name != fname: - newdb.addfile(ti, db.extractfile(ti)) - db.close() - newdb.close() - shutil.move(newdb_path, self.db_path) - except Exception as e: - raise Exception("Removing metadata fail: %s" % e) + self.db.delete(name, version) # removing tarballs - p.arrow("Removing tarballs", 1, self.verbose) + arrow("Removing tarballs", 1, self.verbose) for tb in tbs: - p.arrow("Removing %s" % os.path.basename(tb), 2, self.verbose) + arrow("Removing %s" % os.path.basename(tb), 2, self.verbose) os.unlink(tb) # update last file - p.arrow("Updating last file", 1, self.verbose) + arrow("Updating last file", 1, self.verbose) self.update_last() + + def tarballs(self, name): + '''List all tarballs (script + data)''' + ts = list() + # add script tarballs + ts.append(os.path.abspath(os.path.join(self.image_path, + "%s%s" % (name, Image.image_extension)))) + tempdb = Tarball.open(self.db_path, mode='r:bz2') + jdesc = json.loads(tempdb.get_str("%s.json" % name)) + for dt in jdesc["data"]: + ts.append(os.path.abspath(os.path.join(self.data_path, dt))) + return ts + + +class RepositoryCache(RepositoryBase): + '''Local repository cache class''' + + def __init__(self, cache_path, verbose=True): + self.base_path = os.path.abspath(cache_path) + self.image_path = os.path.join(self.base_path, "image") + self.last_path = os.path.join(self.base_path, "last") + self.db_path = os.path.join(self.base_path, "db") + for path in (self.base_path, self.image_path, self.last_path, self.db_path): + if not os.path.exists(path): + os.mkdir(path) + if not os.access(path, os.W_OK | os.X_OK): + raise Exception("%s is not writable or executable" % path) + self.verbose = verbose + self.repos = dict() + + def register(self, name, image, data): + '''Register a repository to track''' + self.repos[name] = Repository(self.complete_path(image), + self.complete_path(data), + verbose=self.verbose) + + def update(self): + '''Update cache info''' + arrow("Updating repositories", 1, self.verbose) + for r in self.repos: + debug("%s: remote_last: %s, local_last:%s" % (r, + self.repos[r].last(), + self.last(r))) + if self.repos[r].last() > self.last(r): + # copy last file + self.cp(self.repos[r].last_path, os.path.join(self.last_path, r)) + # copy db file + self.cp(self.repos[r].db.path, os.path.join(self.db_path, r)) + arrow("%s updated" % r, 2, self.verbose) + + def last(self, reponame): + '''Return the last timestamp of a repo''' + last_path = os.path.join(self.last_path, reponame) + if os.path.exists(last_path): + return int(open(last_path, "r").read().rstrip()) + return 0 + + def get_image(self, reponame, imagename, imageversion): + '''Obtain a local path in cache for a remote image in repo''' + arrow("Getting image", 1, self.verbose) + filename = "%s-%s%s" % (imagename, imageversion, Image.image_extension) + localpath = os.path.join(self.image_path, filename) + # return db path if exists + if os.path.exists(localpath): + arrow("Found in cache", 2, self.verbose) + return localpath + # get remote image + remotepath = os.path.join(self.repos[reponame].image_path, filename) + arrow("Copying from repository", 2, self.verbose) + self.cp(remotepath, localpath) + return localpath + + def find_image(self, name, version): + '''Find an image in repositories''' + if version is None: + arrow("Serching last version of %s" % name, 1, self.verbose) + else: + arrow("Serching %s version %s " % (name, version), 1, self.verbose) + img = None + # search in all repositories + for repo in self.repos: + tempdb = Database(os.path.join(self.db_path, repo), False) + img = tempdb.find(name, version) + if img is not None: + # \o/ + break + if img is None: + arrow("Not found", 2, self.verbose) + if version is None: + error("Unable to find a version of image %s" % name) + else: + error("Unable to find image %s version %s" % (name, version)) + arrow("Found %s version %s " % (img[0], img[1]), 2, self.verbose) + return (repo, img[0], img[1]) + + def get(self, name, version=None): + '''Return a package object from local cache''' + r, n, v = self.find_image(name, version) + # download image if not in cache + path = self.get_image(r, n, v) + # create an object image + return PackageImage(path, self.verbose) diff --git a/installsystems/tarball.py b/installsystems/tarball.py index d4b108dc9229841d0b71be2865dce487641ae1a1..1c0fe0ff4dedaf9ec30216000ba42a943420dfb0 100644 --- a/installsystems/tarball.py +++ b/installsystems/tarball.py @@ -10,6 +10,7 @@ import os import time import tarfile import StringIO +import re class Tarball(tarfile.TarFile): def add_str(self, name, content, ftype, mode): @@ -27,3 +28,11 @@ class Tarball(tarfile.TarFile): '''Return a string from a filename in a tarball''' ti = self.getmember(name) return self.extractfile(ti).read() + + def getnames(self, reg_pattern=None): + lorig = super(Tarball, self).getnames() + if reg_pattern is None: + return lorig + else: + return [ tpname for tpname in lorig + if re.match(reg_pattern, tpname) ] diff --git a/installsystems/template.py b/installsystems/template.py index 5a7e0108f1202d8b06f7f381c31c3138212326ca..4655c22680db513bcbb52a17bca9eb9f551e8b4f 100644 --- a/installsystems/template.py +++ b/installsystems/template.py @@ -12,9 +12,7 @@ author = parser = """# -*- python -*- # -*- coding: utf-8 -*- -def parser(image): -\t'''Method called by parser''' -\t pass +parser.add_argument("-n", "--hostname", dest="hostname", type=str) # vim:set ts=2 sw=2 noet: """ @@ -22,9 +20,7 @@ def parser(image): setup = """# -*- python -*- # -*- coding: utf-8 -*- -def setup(image): -\t'''Method called by installer''' -\tpass +print "hostname: %s" % args.hostname # vim:set ts=2 sw=2 noet: """