Change db format to sqlite3 (e5e56b89) · Commits · Seblu / installsystems

bin/isinstall

+15 −4

Original line number	Diff line number	Diff line
		@@ -13,7 +13,7 @@ import argparse
		import installsystems
		import installsystems.tools as istools
		from installsystems.printer import *
		from installsystems.repository import RepositoryManager
		from installsystems.repository import RepositoryManager, RepositoryConfig
		from installsystems.image import PackageImage
		from installsystems.config import ConfigFile

		@@ -34,12 +34,16 @@ p_main.add_argument('-d', "--debug", action=DebugAction, nargs=0,
		help="active debug mode")
		p_main.add_argument('-q', "--quiet", action="store_false", dest="verbose", default=True,
		help="active quiet mode")
		p_main.add_argument("--no-cache", action="store_false", default=False,
		help="Not use persistent db caching")
		p_main.add_argument("-c", "--config", dest="config", type=str, default=None,
		help="config file path")
		p_main.add_argument("-v", "--image-version", dest="image_version", type=int, default=None,
		help="image version")
		p_main.add_argument("-t", "--timeout", dest="timeout", type=int, default=None,
		help="download timeout")
		p_main.add_argument("-r", "--repo", dest="repos", action="append", default=[],
		help="repository (can be specified more than one time)")
		p_main.add_argument("image_name", type=str,
		help="image to install (path or name)")
		try:
		@@ -52,10 +56,17 @@ try:
		if image_name_type == "file":
		pkg = PackageImage(istools.abspath(args.image_name))
		elif image_name_type == "name":
		# remove cache is asked
		if args.no_cache:
		config.cache = None
		# init repo cache object
		repoman = RepositoryManager(timeout=args.timeout, verbose=args.verbose)
		# register repositories
		repoman.register(config.repos)
		repoman = RepositoryManager(config.cache, timeout=args.timeout, verbose=args.verbose)
		# register config repositories
		for crepo in config.repos:
		repoman.register(crepo)
		# register command line repositories
		for rpath in args.repos:
		repoman.register(RepositoryConfig(None, path=rpath))
		# get image package
		pkg = repoman.get(args.image_name, args.image_version)
		else:

bin/isrepo

+4 −5

Original line number	Diff line number	Diff line
		@@ -26,24 +26,23 @@ def init(args):
		'''Create an empty fresh repo tree'''
		# call init from library
		try:
		Repository.create(args.repository, args.verbose)
		Repository.create(args.repository, verbose=args.verbose)
		except Exception as e:
		raise Exception("init failed: %s" % e)

		def add(args):
		'''Add a package to repository'''
		try:
		repo = Repository(args.repository, args.verbose)
		pkg = PackageImage(args.path, args.verbose)
		repo = Repository(args.repository, verbose=args.verbose)
		pkg = PackageImage(args.path, verbose=args.verbose)
		repo.add(pkg)
		except Exception as e:
		raise
		raise Exception("add failed: %s" % e)

		def delete(args):
		'''Remove a package from repository'''
		try:
		repo = Repository(args.repository, args.verbose)
		repo = Repository(args.repository, verbose=args.verbose)
		repo.delete(args.image_name, args.image_version)
		except Exception as e:
		raise Exception("del failed: %s" % e)

installsystems/config.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -58,7 +58,7 @@ class ConfigFile(object):

		def _cache_paths(self):
		'''List all candidates to cache directories. Alive or not'''
		dirs = ["/var/tmp", "/tmp"]
		dirs = [] # ["/var/tmp", "/tmp"]
		# we have a different behaviour if we are root
		dirs.insert(0, "/var/cache" if os.getuid() == 0 else os.path.expanduser("~/.cache"))
		return map(lambda x: os.path.join(x, self.prefix), dirs)

installsystems/database.py

+50 −49

Original line number	Diff line number	Diff line
		@@ -11,12 +11,16 @@ import os
		import shutil
		import tarfile
		import cStringIO
		import sqlite3
		import installsystems.tools as istools
		import installsystems.template as istemplate
		from installsystems.tarball import Tarball
		from installsystems.printer import *

		class Database(object):
		'''Abstract repo database stuff'''
		''' Abstract repo database stuff
		It needs to be local cause of sqlite3 which need to open a file
		'''

		db_format = "1"

		@@ -26,23 +30,27 @@ class Database(object):
		# check locality
		if istools.pathtype(path) != "file":
		raise NotImplementedError("Database creation must be local")
		dbpath = istools.abspath(path)
		if os.path.exists(dbpath):
		path = os.path.abspath(path)
		if os.path.exists(path):
		raise Exception("Database already exists. Remove it before")
		try:
		tarball = Tarball.open(dbpath, mode="w:gz", dereference=True)
		tarball.add_str("format", Database.db_format, tarfile.REGTYPE, 0444)
		tarball.close()
		conn = sqlite3.connect(path, isolation_level=None)
		conn.execute("PRAGMA foreign_keys = ON")
		conn.executescript(istemplate.createdb)
		conn.commit()
		conn.close()
		except Exception as e:
		raise Exception("Create database failed: %s" % e)
		return cls(path, verbose)

		def __init__(self, path, verbose=True):
		self.path = istools.abspath(path)
		# check locality
		if istools.pathtype(path) != "file":
		raise NotImplementedError("Database creation must be local")
		self.path = os.path.abspath(path)
		self.verbose = verbose
		# load db in memory
		self.file = cStringIO.StringIO()
		shutil.copyfileobj(istools.uopen(self.path), self.file)
		self.conn = sqlite3.connect(self.path, isolation_level=None)
		self.conn.execute("PRAGMA foreign_keys = ON")

		def get(self, name, version):
		'''Return a description dict from a package name'''
		@@ -55,44 +63,48 @@ class Database(object):
		except KeyError:
		raise Exception("No package %s version %s in metadata" % (name, version))
		except Exception as e:
		raise Exception("Unable to read db %s version %s: e" % (name, version, e))
		raise Exception("Unable to read db %s version %s: %s" % (name, version, e))
		# convert loaded data into dict (json parser)
		try:
		return json.loads(rdata)
		except Exception as e:
		raise Exception("Invalid metadata in package %s version %s: e" % (name, version, e))

		def ask(self, sql, args=()):
		'''Ask question to db'''
		return self.conn.execute(sql, args)

		def add(self, package):
		'''Add a packaged image to a db'''
		arrow("Adding metadata to db", 1, self.verbose)
		# check locality
		if istools.pathtype(self.path) != "file":
		raise NotImplementedError("Database addition must be local")
		# naming
		newdb_path = "%s.new" % self.path
		# compute md5
		arrow("Formating metadata", 2, self.verbose)
		desc = package.description
		desc["md5"] = package.md5
		jdesc = json.dumps(desc)
		try:
		arrow("Adding metadata", 2, self.verbose)
		self.file.seek(0)
		newfile = cStringIO.StringIO()
		db = Tarball.open(fileobj=self.file, mode='r:gz')
		newdb = Tarball.open(fileobj=newfile, mode='w:gz')
		for ti in db.getmembers():
		if ti.name != package.id:
		newdb.addfile(ti, db.extractfile(ti))
		newdb.add_str(package.id, jdesc, tarfile.REGTYPE, 0644)
		db.close()
		newdb.close()
		# writing to disk
		arrow("Writing to disk", 2, self.verbose)
		self.file.close()
		self.file = newfile
		self.write()
		# let's go
		arrow("Begin transaction to db", 1, self.verbose)
		self.conn.execute("BEGIN TRANSACTION")
		# insert image information
		arrow("Add image metadata", 2, self.verbose)
		self.conn.execute("INSERT OR REPLACE INTO image values (?,?,?,?,?,?,?)",
		(package.md5,
		package.name,
		package.version,
		package.date,
		package.author,
		package.description,
		package.size,
		))
		# insert data informations
		arrow("Add data metadata", 2, self.verbose)
		for key,value in package.data.items():
		self.conn.execute("INSERT OR REPLACE INTO data values (?,?,?,?)",
		(value["md5"],
		package.md5,
		key,
		value["size"]
		))
		# on commit
		arrow("Commit transaction to db", 1, self.verbose)
		self.conn.execute("COMMIT TRANSACTION")
		except Exception as e:
		raise
		raise Exception("Adding metadata fail: %s" % e)

		def delete(self, name, version):
		@@ -148,14 +160,3 @@ class Database(object):
		if int(version) not in candidates:
		return None
		return self.get(name, version)

		def write(self):
		'''Write current dabatase into its file'''
		if istools.pathtype(self.path) != "file":
		raise NotImplementedError("Database writing must be local")
		try:
		dest = open(self.path, "w")
		self.file.seek(0)
		shutil.copyfileobj(self.file, dest)
		except Exception as e:
		raise Exception("Unable to write database: %s" % e)

installsystems/image.py

+63 −57

Original line number	Diff line number	Diff line
		@@ -240,88 +240,94 @@ class SourceImage(Image):
		class PackageImage(Image):
		'''Packaged image manipulation class'''

		def __init__(self, path, verbose=True):
		def __init__(self, path, md5name=False, verbose=True):
		Image.__init__(self)
		self.path = istools.abspath(path)
		self.base_path = os.path.dirname(self.path)
		self.verbose = verbose
		# tarball are named by md5 and not by real name
		self.md5name = md5name
		# load image in memory
		arrow("Loading tarball in memory", 1, verbose)
		self.file = cStringIO.StringIO()
		memfile = cStringIO.StringIO()
		fo = istools.uopen(self.path)
		shutil.copyfileobj(fo, self.file)
		(self.size, self.md5) = istools.copyfileobj(fo, memfile)
		fo.close()
		# set tarball fo
		self.file.seek(0)
		self.tarball = Tarball.open(fileobj=self.file, mode='r:gz')
		self.parse()
		self._md5 = None
		memfile.seek(0)
		self._tarball = Tarball.open(fileobj=memfile, mode='r:gz')
		self._metadata = self.read_metadata()

		@property
		def md5(self):
		'''Return md5sum of the current tarball'''
		if self._md5 is None:
		self._md5 = istools.md5sum(self.path)
		return self._md5
		def __getattr__(self, name):
		"""Give direct access to description field"""
		if name in self._metadata:
		return self._metadata[name]
		raise AttributeError

		@property
		def id(self):
		'''Return image versionned name / id'''
		return "%s-%s" % (self.description["name"], self.description["version"])

		@property
		def name(self):
		'''Return image name'''
		return self.description["name"]

		@property
		def version(self):
		'''Return image version'''
		return self.description["version"]
		return "%s-%s" % (self._metadata["name"], self._metadata["version"])

		@property
		def filename(self):
		'''Return image filename'''
		return "%s%s" % (self.id, self.extension)

		@property
		def datas(self):
		'''Create a dict of data tarballs'''
		d = dict()
		for dt in self.description["data"]:
		d[dt] = dict(self.description["data"][dt])
		d[dt]["filename"] = "%s-%s%s" % (self.id, dt, self.extension_data)
		d[dt]["path"] = os.path.join(self.base_path, d[dt]["filename"])
		return d

		def parse(self):
		'''Parse tarball and extract metadata'''
		def read_metadata(self):
		'''Parse tarball and return metadata dict'''
		# extract metadata
		arrow("Read tarball metadata", 1, self.verbose)
		img_format = self.tarball.get_str("format")
		img_desc = self.tarball.get_str("description.json")
		img_format = self._tarball.get_str("format")
		img_desc = self._tarball.get_str("description.json")
		# check format
		arrow("Read format", 2, self.verbose)
		arrow("Read format file", 2, self.verbose)
		if img_format != self.format:
		raise Exception("Invalid tarball image format")
		# check description
		arrow("Read description", 2, self.verbose)
		arrow("Read description file", 2, self.verbose)
		try:
		self.description = json.loads(img_desc)
		desc = json.loads(img_desc)
		except Exception as e:
		raise Exception("Invalid description: %s" % e1)
		# FIXME: we should check valid information here
		return desc

		def check_md5(self):
		'''Check if md5 of data tarballs are correct'''
		arrow("Check MD5", 1, self.verbose)
		databalls = self.description["data"]
		for databall in databalls:
		md5_path = os.path.join(self.base_path, databall)
		arrow(os.path.relpath(md5_path), 2, self.verbose)
		md5_meta = databalls[databall]["md5"]
		md5_file = istools.md5sum(md5_path)
		if md5_meta != md5_file:
		raise Exception("Invalid md5: %s" % databall)
		@property
		def tarballs(self):
		'''List path of all related tarballs'''
		d_d = {}
		name = os.path.join(self.base_path, self.md5) if self.md5name else self.path
		d_d[name] = {"md5": self.md5, "size": self.size}
		for key, value in self._metadata["data"].items():
		if self.md5name:
		name = os.path.join(self.base_path, value["md5"])
		else:
		name = os.path.join(self.base_path,
		"%s-%s%s" % (self.id, key, self.extension_data))
		d_d[name] = {"md5": value["md5"], "size": value["size"]}
		return d_d

		def tarcheck(self, message="Check MD5"):
		'''Check md5 and size of tarballs are correct'''
		arrow(message, 1, self.verbose)
		# open /dev/null
		dn = open("/dev/null", "w")
		for key,value in self.tarballs.items():
		arrow(os.path.basename(key), 2, self.verbose)
		# open tarball
		tfo = istools.uopen(key)
		# compute sum and md5 using copy function
		size, md5 = istools.copyfileobj(tfo ,dn)
		# close tarball fo
		tfo.close()
		# check md5
		if md5 != value["md5"]:
		raise Exception("Invalid md5: %s" % key)
		# check size
		if size != value["size"]:
		raise Exception("Invalid size: %s" % key)
		dn.close()

		def run_parser(self, gl):
		'''Run parser scripts'''
		@@ -336,14 +342,14 @@ class PackageImage(Image):
		'''Run scripts in a tarball directory'''
		arrow("Run %s" % directory, 1, self.verbose)
		# get list of parser scripts
		l_scripts = self.tarball.getnames("%s/.*\.py" % directory)
		l_scripts = self._tarball.getnames("%s/.*\.py" % directory)
		# order matter!
		l_scripts.sort()
		# run scripts
		for n_scripts in l_scripts:
		arrow(os.path.basename(n_scripts), 2, self.verbose)
		try:
		s_scripts = self.tarball.get_str(n_scripts)
		s_scripts = self._tarball.get_str(n_scripts)
		except Exception as e:
		raise Exception("Extracting script %s fail: %s" %
		(os.path.basename(n_scripts), e))
		@@ -357,10 +363,10 @@ class PackageImage(Image):
		def extractdata(self, dataname, target, filelist=None):
		'''Extract a data tarball into target'''
		# check if dataname exists
		if dataname not in self.description["data"].keys():
		raise Exception("No such data")
		if dataname not in self._metadata["data"].keys():
		raise Exception("No such data: %s" % dataname)
		# tarball info
		tinfo = self.description["data"][dataname]
		tinfo = self._metadata["data"][dataname]
		# build data tar paths
		paths = [ os.path.join(self.base_path, tinfo["md5"]),
		os.path.join(self.base_path, "%s-%s%s" % (self.id,