Update repository version (857d967c) · Commits · Seblu / installsystems

installsystems/database.py

+10 −5

Original line number	Diff line number	Diff line
		@@ -20,8 +20,10 @@
		Database stuff
		'''

		import math
		import os
		import sqlite3
		import uuid
		import installsystems.tools as istools
		import installsystems.template as istemplate
		from installsystems.tarball import Tarball
		@@ -34,6 +36,8 @@ class Database(object):
		It needs to be local cause of sqlite3 which need to open a file
		'''

		version = 2.0

		@classmethod
		def create(cls, path):
		arrow("Creating repository database")
		@@ -47,6 +51,8 @@ class Database(object):
		conn = sqlite3.connect(path, isolation_level=None)
		conn.execute("PRAGMA foreign_keys = ON")
		conn.executescript(istemplate.createdb)
		conn.execute("INSERT INTO repository values (?,?,?)",
		(str(uuid.uuid4()), Database.version, "",))
		conn.commit()
		conn.close()
		except Exception as e:
		@@ -64,16 +70,15 @@ class Database(object):
		self.conn.execute("PRAGMA foreign_keys = ON")
		# get database version
		try:
		r = self.ask("SELECT value FROM misc WHERE key = 'version'").fetchone()
		r = self.ask("SELECT version FROM repository").fetchone()
		if r is None:
		raise TypeError()
		self.version = float(r[0])
		except:
		self.version = 1.0
		# we only support database v1
		if self.version >= 2.0:
		debug(u"Invalid database format: %s" % self.version)
		raise ISError("Invalid database format")
		if math.floor(self.version) >= math.floor(Database.version) + 1.0:
		raise ISWarning(u"New database format (%s), please upgrade "
		"your Installsystems version" % self.version)
		# we make a query to be sure format is valid
		try:
		self.ask("SELECT * FROM image")

installsystems/repository.py

+165 −43

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ import tempfile
		import fnmatch
		import cStringIO
		import json
		import uuid
		import installsystems
		import installsystems.tools as istools
		from installsystems.exception import *
		@@ -39,6 +40,35 @@ from installsystems.tools import PipeFile
		from installsystems.image import Image, PackageImage
		from installsystems.database import Database

		class RepositoryFactory(object):
		'''
		Repository factory
		'''

		def __init__(self):
		self.repo_class = {
		1: Repository_v1,
		2: Repository
		}

		def create(self, config):
		db = None
		if not config.offline:
		try:
		db = Database(config.dbpath)
		except ISWarning as e:
		warn('[%s]: %s' % (config.name, e))
		config.offline = True
		except ISError:
		debug(u"Unable to load database %s" % config.dbpath)
		config.offline = True
		if config.offline:
		debug(u"Repository %s is offline" % config.name)
		if db is None:
		return Repository(config)
		else:
		return self.repo_class[int(db.version)](config, db)

		class Repository(object):
		'''
		Repository class
		@@ -113,17 +143,10 @@ class Repository(object):
		for md5 in i_only2: pimg(repo2, "g", md5, i_dict2)
		for md5 in p_only2: ppay(repo2, "g", md5, p_dict2)

		def __init__(self, config):
		def __init__(self, config, db=None):
		self.config = config
		self.local = istools.isfile(self.config.path)
		if not self.config.offline:
		try:
		self.db = Database(config.dbpath)
		except:
		debug(u"Unable to load database %s" % config.dbpath)
		self.config.offline = True
		if self.config.offline:
		debug(u"Repository %s is offline" % config.name)
		self.db = db

		def __getattribute__(self, name):
		'''
		@@ -147,6 +170,13 @@ class Repository(object):
		'''
		return self.db.version

		@property
		def uuid(self):
		'''
		Return repository UUID
		'''
		return self.db.ask("SELECT uuid from repository").fetchone()[0]

		def init(self):
		'''
		Initialize an empty base repository
		@@ -206,6 +236,40 @@ class Repository(object):
		# return last
		return reduce(f, r)

		def _add(self, image):
		'''
		Add description to db
		'''
		arrow("Adding metadata")
		self.db.begin()
		# insert image information
		arrow("Image", 1)
		self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?,?,?)",
		(image.md5,
		image.name,
		image.version,
		image.date,
		image.author,
		image.description,
		image.size,
		image.is_min_version,
		image.img_format,
		))
		# insert data information
		arrow("Payloads", 1)
		for name, obj in image.payload.items():
		self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
		(obj.md5,
		image.md5,
		name,
		obj.isdir,
		obj.size,
		))
		# on commit
		self.db.commit()
		# update last file
		self.update_last()

		def add(self, image, delete=False):
		'''
		Add a packaged image to repository
		@@ -240,34 +304,7 @@ class Repository(object):
		r_image.md5 = image.md5
		# checking image and payload after copy
		r_image.check("Check image and payload")
		# add description to db
		arrow("Adding metadata")
		self.db.begin()
		# insert image information
		arrow("Image", 1)
		self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?)",
		(image.md5,
		image.name,
		image.version,
		image.date,
		image.author,
		image.description,
		image.size,
		))
		# insert data informations
		arrow("Payloads", 1)
		for name, obj in image.payload.items():
		self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
		(obj.md5,
		image.md5,
		name,
		obj.isdir,
		obj.size,
		))
		# on commit
		self.db.commit()
		# update last file
		self.update_last()
		self._add(image)
		# removing orginal files
		if delete:
		arrow("Removing original files")
		@@ -497,6 +534,70 @@ class Repository(object):
		(a[0],)).fetchall()
		return [ a[0] ] + [ x[0] for x in b ]

		class Repository_v1(Repository):

		def _add(self, image):
		'''
		Add description to db
		'''
		arrow("Adding metadata")
		self.db.begin()
		# insert image information
		arrow("Image", 1)
		self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?)",
		(image.md5,
		image.name,
		image.version,
		image.date,
		image.author,
		image.description,
		image.size,
		))
		# insert data information
		arrow("Payloads", 1)
		for name, obj in image.payload.items():
		self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
		(obj.md5,
		image.md5,
		name,
		obj.isdir,
		obj.size,
		))
		# on commit
		self.db.commit()
		# update last file
		self.update_last()

		@property
		def uuid(self):
		'''
		Return repository UUID
		'''
		# repository v1 don't have UUID, generate one with repository url
		# encode in ascii for hash digest
		return str(uuid.uuid5(uuid.NAMESPACE_URL,
		self.config.path.encode('ascii')))

		def images(self):
		'''
		Return a dict of information on images
		'''
		db_images = self.db.ask("SELECT md5, name, version, date, author, \
		description, size \
		FROM image ORDER BY name, version").fetchall()

		images = []
		field = ("md5", "name", "version", "date", "author", "description",
		"size")
		for info in db_images:
		d = dict(zip(field, info))
		d["repo"] = self.config.name
		d["url"] = os.path.join(self.config.path, d["md5"])
		d["format"] = 1
		d["is_min_version"] = 9
		images.append(d)
		return images


		class RepositoryManager(object):
		'''
		@@ -512,6 +613,7 @@ class RepositoryManager(object):
		self.filter = [] if filter is None else filter
		self.search = [] if search is None else search
		self.timeout = timeout or 3
		self.factory = RepositoryFactory()
		debug(u"Repository timeout setted to %ds" % self.timeout)
		if cache_path is None:
		self.cache_path = None
		@@ -551,9 +653,19 @@ class RepositoryManager(object):
		if isinstance(key, int):
		return self.repos[key]
		elif isinstance(key, basestring):
		# match name
		for repo in self.repos:
		if repo.config.name == key:
		return repo
		# if not found, match uuid
		# we need at least 4 chars to avoid ambiguous uuid matching
		if len(key) >= 4:
		for repo in [r for r in self.repos if not r.config.offline]:
		if fnmatch.fnmatch(repo.uuid, "%s*" % key):
		return repo
		else:
		raise ISWarning("Ambiguous argument: we need at least 4 chars "
		"to match an uuid")
		raise IndexError(u"No repository named: %s" % key)
		else:
		raise TypeError(u"Invalid type %s for %s" % (type(key), key))
		@@ -584,11 +696,11 @@ class RepositoryManager(object):
		debug(u"Registering offline repository %s (%s)" % (config.path, config.name))
		# we must force offline in cast of argument offline
		config.offline = True
		self.repos.append(Repository(config))
		self.repos.append(self.factory.create(config))
		# if path is local, no needs to create a cache
		elif istools.isfile(config.path):
		debug(u"Registering direct repository %s (%s)" % (config.path, config.name))
		self.repos.append(Repository(config))
		self.repos.append(self.factory.create(config))
		# path is remote, we need to create a cache
		else:
		debug(u"Registering cached repository %s (%s)" % (config.path, config.name))
		@@ -655,7 +767,7 @@ class RepositoryManager(object):
		# if something append bad during caching, we mark repo as offline
		debug(u"Unable to cache repository %s: %s" % (config.name, e))
		config.offline = True
		return Repository(config)
		return self.factory.create(config)

		@property
		def names(self):
		@@ -702,7 +814,12 @@ class RepositoryManager(object):
		# No path means only in searchable repositories
		if path is None:
		for k, v in images.items():
		# match name
		if v["repo"] not in self.search:
		uuid = self[v["repo"]].uuid
		# match uuid
		if not [uuid for pat in self.search
		if fnmatch.fnmatch(uuid, '%s*' % pat)]:
		del images[k]
		path = "*"
		# No version means last version
		@@ -717,10 +834,15 @@ class RepositoryManager(object):
		versions.remove(last)
		for rmv in versions:
		del images[u"%s/%s:%s" % (repo, img, rmv)]
		# if 'path*' do not match a repo name, it may be an uuid, so add
		# globbing for smart uuid matching
		if not fnmatch.filter(self.onlines, "%s*" % path):
		path = "%s*" % path
		# filter with pattern on path
		filter_pattern = u"%s/%s:%s" % (path, image, version)
		for k in images.keys():
		if not fnmatch.fnmatch(k, filter_pattern):
		for k, img in images.items():
		if not (fnmatch.fnmatch(k, filter_pattern) or
		fnmatch.fnmatch("%s/%s" % (self[img["repo"]].uuid, k.split("/")[1]), filter_pattern)):
		del images[k]
		ans.update(images)
		return ans

installsystems/template.py

+11 −5

Original line number	Diff line number	Diff line
		@@ -90,6 +90,8 @@ CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
		author TEXT,
		description TEXT,
		size INTEGER NOT NULL,
		is_min_version INTEGER NOT NULL,
		format INTEGER NOT NULL,
		UNIQUE(name, version));

		CREATE TABLE payload (md5 TEXT NOT NULL,
		@@ -98,4 +100,8 @@ CREATE TABLE payload (md5 TEXT NOT NULL,
		isdir INTEGER NOT NULL,
		size INTEGER NOT NULL,
		PRIMARY KEY(md5, image_md5));

		CREATE TABLE repository (uuid TEXT NOT NULL PRIMARY KEY,
		version FLOAT NOT NULL,
		motd TEXT NOT NULL);
		"""