diff --git a/bin/is b/bin/is
index 4df656a2c4cdcecf345d2b4bb7d7d9d772ea41e6..e65938a276eff72cd7a30fdeb8d1ece454fca3d4 100755
--- a/bin/is
+++ b/bin/is
@@ -28,7 +28,9 @@ from installsystems.exception import ISError, ISException
from installsystems.image import PackageImage, SourceImage
from installsystems.printer import arrow, arrowlevel, setmode
from installsystems.printer import out, warn, error, debug, confirm
-from installsystems.repository import Repository, RepositoryManager, RepositoryConfig
+from installsystems.repository import split_list as split_repo_list, diff as repodiff
+from installsystems.repository.manager import RepositoryManager
+from installsystems.repository.config import RepositoryConfig
from installsystems.tools import chroot, prepare_chroot, unprepare_chroot
from installsystems.tools import islocal, smd5sum, argv
from os import getpid, getcwdu, chdir
@@ -48,8 +50,8 @@ def load_repositories(args):
if args.no_cache:
args.cache = None
# split filter and search in list
- args.repo_filter = Repository.split_list(args.repo_filter)
- args.repo_search = Repository.split_list(args.repo_search)
+ args.repo_filter = split_repo_list(args.repo_filter)
+ args.repo_search = split_repo_list(args.repo_search)
# init repo cache object
repoman = RepositoryManager(args.cache, timeout=args.repo_timeout or args.timeout,
filter=args.repo_filter, search=args.repo_search)
@@ -252,7 +254,7 @@ def c_diff(args):
repoman = load_repositories(args)
if args.object[0] in repoman.onlines and args.object[1] in repoman.onlines:
try:
- Repository.diff(repoman[args.object[0]], repoman[args.object[1]])
+ diff(repoman[args.object[0]], repoman[args.object[1]])
except IndexError as e:
raise ISError(e)
else:
diff --git a/installsystems/image/package.py b/installsystems/image/package.py
index 029cb74ef15edc291eba700142acbb3cc7327a52..83bccc9826e29457ed0599d02468c5bfbfeee575 100644
--- a/installsystems/image/package.py
+++ b/installsystems/image/package.py
@@ -34,7 +34,7 @@ from installsystems.tools import mkdir, abspath, time_rfc2822, human_size, argv,
from json import loads, dumps
from math import floor
from os import listdir
-from os.path import join, basename, exists, isdir, dirname, abspath
+from os.path import join, basename, exists, isdir, dirname
from time import time
class PackageImage(Image):
diff --git a/installsystems/repository/__init__.py b/installsystems/repository/__init__.py
index ea33e7fbe550f201557074f4a22fd2b6acaa5147..0e39cab7c8b47d97adef69196b67bbc2d43198d7 100644
--- a/installsystems/repository/__init__.py
+++ b/installsystems/repository/__init__.py
@@ -20,8 +20,80 @@
InstallSystems repository package
'''
-from installsystems.repository.manager import RepositoryManager
-from installsystems.repository.config import RepositoryConfig
-from installsystems.repository.repository import Repository
-from installsystems.repository.repository1 import Repository1
-from installsystems.repository.repository2 import Repository2
+__all__ = [
+ "is_name",
+ "check_name",
+ "split_path",
+ "split_list",
+ "diff",
+]
+
+from installsystems.exception import ISError
+from installsystems.printer import arrow, out
+from re import match, split
+
+def is_name(name):
+ '''Check if name is a valid repository name'''
+ return match("^[-_\w]+$", name) is not None
+
+def check_name(name):
+ '''
+ Raise exception is repository name is invalid
+ '''
+ if not is_name(name):
+ raise ISError(u"Invalid repository name %s" % name)
+ return name
+
+def split_path(path):
+ '''
+ Split an image path (repo/image:version)
+ in a tuple (repo, image, version)
+ '''
+ x = match(u"^(?:([^/:]+)/)?([^/:]+)?(?::v?([^/:]+)?)?$", path)
+ if x is None:
+ raise ISError(u"invalid image path: %s" % path)
+ return x.group(1, 2, 3)
+
+def split_list(repolist, filter=None):
+ '''
+ Return a list of repository from a comma/spaces separated names of repo
+ '''
+ if filter is None:
+ filter = is_name
+ return [r for r in split("[ ,\n\t\v]+", repolist) if filter(r)]
+
+@staticmethod
+def diff(repo1, repo2):
+ '''
+ Compute a diff between two repositories
+ '''
+ arrow(u"Diff between repositories #y#%s#R# and #g#%s#R#" % (repo1.config.name,
+ repo2.config.name))
+ # Get info from databases
+ i_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
+ "SELECT md5, name, version FROM image").fetchall())
+ i_set1 = set(i_dict1.keys())
+ i_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
+ "SELECT md5, name, version FROM image").fetchall())
+ i_set2 = set(i_dict2.keys())
+ p_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
+ "SELECT md5, name FROM payload").fetchall())
+ p_set1 = set(p_dict1.keys())
+ p_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
+ "SELECT md5, name FROM payload").fetchall())
+ p_set2 = set(p_dict2.keys())
+ # computing diff
+ i_only1 = i_set1 - i_set2
+ i_only2 = i_set2 - i_set1
+ p_only1 = p_set1 - p_set2
+ p_only2 = p_set2 - p_set1
+ # printing functions
+ pimg = lambda r,c,m,d,: out("#%s#Image only in repository %s: %s v%s (%s)#R#" %
+ (c, r.config.name, d[m][0], d[m][1], m))
+ ppay = lambda r,c,m,d,: out("#%s#Payload only in repository %s: %s (%s)#R#" %
+ (c, r.config.name, d[m][0], m))
+ # printing image diff
+ for md5 in i_only1: pimg(repo1, "y", md5, i_dict1)
+ for md5 in p_only1: ppay(repo1, "y", md5, p_dict1)
+ for md5 in i_only2: pimg(repo2, "g", md5, i_dict2)
+ for md5 in p_only2: ppay(repo2, "g", md5, p_dict2)
diff --git a/installsystems/repository/config.py b/installsystems/repository/config.py
index c71dedf023911e73b58733fe54aa0a5662fa930f..35fc3e4eae76230ee9869d301b34216fe14ade84 100644
--- a/installsystems/repository/config.py
+++ b/installsystems/repository/config.py
@@ -22,8 +22,8 @@ Repository configuration module
from grp import getgrnam
from installsystems.printer import warn, debug
-from installsystems.repository.repository import Repository
-from installsystems.tools import isfile, chrights, mkdir, compare_versions
+from installsystems.repository import check_name
+from installsystems.tools import islocal, chrights, mkdir, compare_versions
from os import getuid, getgid, umask, linesep
from os.path import join, abspath
from pwd import getpwnam
@@ -37,7 +37,7 @@ class RepositoryConfig(object):
# set default value for arguments
self._valid_param = ("name", "path", "dbpath", "lastpath",
"uid", "gid", "fmod", "dmod", "offline")
- self.name = Repository.check_name(name)
+ self.name = check_name(name)
self.path = ""
self._offline = False
self._dbpath = None
@@ -111,7 +111,7 @@ class RepositoryConfig(object):
Set db path
'''
# dbpath must be local, sqlite3 requirement
- if not isfile(value):
+ if not islocal(value):
raise ValueError("Database path must be local")
self._dbpath = abspath(value)
diff --git a/installsystems/repository/database.py b/installsystems/repository/database.py
deleted file mode 100644
index ba9934ecbf51d8fb520a4696fef5168845d36ba2..0000000000000000000000000000000000000000
--- a/installsystems/repository/database.py
+++ /dev/null
@@ -1,129 +0,0 @@
-# -*- python -*-
-# -*- coding: utf-8 -*-
-
-# This file is part of Installsystems.
-#
-# Installsystems is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Installsystems is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Installsystems. If not, see .
-
-'''
-Database stuff
-'''
-
-import math
-import os
-import sqlite3
-import uuid
-import installsystems.tools as istools
-from installsystems.exception import *
-from installsystems.printer import *
-
-class Database(object):
- '''
- Abstract repo database stuff
- It needs to be local cause of sqlite3 which need to open a file
- '''
-
- version = 2.0
-
- @classmethod
- def create(cls, path):
- arrow("Creating repository database")
- # check locality
- if not istools.isfile(path):
- raise ISError("Database creation must be local")
- path = os.path.abspath(path)
- if os.path.exists(path):
- raise ISError("Database already exists. Remove it before")
- try:
- conn = sqlite3.connect(path, isolation_level=None)
- conn.execute("PRAGMA foreign_keys = ON")
- conn.executescript(TEMPLATE_EMPTY_DB)
- conn.execute("INSERT INTO repository values (?,?,?)",
- (str(uuid.uuid4()), Database.version, "",))
- conn.commit()
- conn.close()
- except Exception as e:
- raise ISError(u"Create database failed", e)
- return cls(path)
-
- def __init__(self, path):
- # check locality
- if not istools.isfile(path):
- raise ISError("Database must be local")
- self.path = os.path.abspath(path)
- if not os.path.exists(self.path):
- raise ISError("Database not exists")
- self.conn = sqlite3.connect(self.path, isolation_level=None)
- self.conn.execute("PRAGMA foreign_keys = ON")
- # get database version
- try:
- r = self.ask("SELECT version FROM repository").fetchone()
- if r is None:
- raise TypeError()
- self.version = float(r[0])
- except:
- self.version = 1.0
- if math.floor(self.version) >= math.floor(Database.version) + 1.0:
- raise ISWarning(u"New database format (%s), please upgrade "
- "your Installsystems version" % self.version)
- # we make a query to be sure format is valid
- try:
- self.ask("SELECT * FROM image")
- except:
- debug(u"Invalid database format: %s" % self.version)
- raise ISError("Invalid database format")
-
- def begin(self):
- '''
- Start a db transaction
- '''
- self.conn.execute("BEGIN TRANSACTION")
-
- def commit(self):
- '''
- Commit current db transaction
- '''
- self.conn.execute("COMMIT TRANSACTION")
-
-
- def ask(self, sql, args=()):
- '''
- Ask question to db
- '''
- return self.conn.execute(sql, args)
-
-
-TEMPLATE_EMPTY_DB = u"""
-CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
- name TEXT NOT NULL,
- version TEXT NOT NULL,
- date INTEGER NOT NULL,
- author TEXT,
- description TEXT,
- size INTEGER NOT NULL,
- is_min_version INTEGER NOT NULL,
- format INTEGER NOT NULL,
- UNIQUE(name, version));
-
-CREATE TABLE payload (md5 TEXT NOT NULL,
- image_md5 TEXT NOT NULL REFERENCES image(md5),
- name TEXT NOT NULL,
- isdir INTEGER NOT NULL,
- size INTEGER NOT NULL,
- PRIMARY KEY(md5, image_md5));
-
-CREATE TABLE repository (uuid TEXT NOT NULL PRIMARY KEY,
- version FLOAT NOT NULL,
- motd TEXT NOT NULL);
-"""
diff --git a/installsystems/repository/factory.py b/installsystems/repository/factory.py
index 2d920b45f7dfbb1be40645a814cf7c85f0acaaa9..3539e2253f833f7c73288c912bde0283e1b39f42 100644
--- a/installsystems/repository/factory.py
+++ b/installsystems/repository/factory.py
@@ -20,9 +20,7 @@
Repository Factory
'''
-from installsystems.printer import debug, warn
-from installsystems.exception import ISWarning, ISError
-from installsystems.repository.database import Database
+from installsystems.exception import ISError
from installsystems.repository.repository1 import Repository1
from installsystems.repository.repository2 import Repository2
@@ -31,28 +29,21 @@ class RepositoryFactory(object):
Repository factory
'''
- def __init__(self):
-
- self.repo_class = {
- 1: Repository1,
- 2: Repository2,
- }
-
- def create(self, config):
- db = None
- if not config.offline:
- try:
- db = Database(config.dbpath)
- except ISWarning as e:
- warn('[%s]: %s' % (config.name, e))
- config.offline = True
- except ISError:
- debug(u"Unable to load database %s" % config.dbpath)
- config.offline = True
- if config.offline:
- debug(u"Repository %s is offline" % config.name)
- if db is None:
+ def __new__(cls, config):
+ '''
+ Factory design pattern.
+ Return the right object version based on a version detector function
+ '''
+ version = cls.version(config.dbpath)
+ if version == 1:
+ return Repostory1(config)
+ elif version == 2:
return Repository2(config)
- else:
- return self.repo_class[int(db.version)](config, db)
-
+ raise ISError(u"Unsupported repository version")
+
+ @staticmethod
+ def version(path):
+ '''
+ Return the version of a database
+ '''
+ return 2
diff --git a/installsystems/repository/manager.py b/installsystems/repository/manager.py
index 7023b4868cf505d047514cbe508d05a81f38334c..17f5cd3c2d96fe6f9bc55b4522fc0dab40dc667e 100644
--- a/installsystems/repository/manager.py
+++ b/installsystems/repository/manager.py
@@ -22,9 +22,9 @@ Repository management module
from installsystems.exception import ISError, ISWarning
from installsystems.printer import out, debug, arrow
+from installsystems.repository import split_path
from installsystems.repository.factory import RepositoryFactory
-from installsystems.repository.repository import Repository
-from installsystems.tools import isfile, chrights, PipeFile, compare_versions
+from installsystems.tools import islocal, chrights, PipeFile, compare_versions
from installsystems.tools import time_rfc2822, human_size, strcspn
from json import dumps
from os import mkdir, access, W_OK, X_OK, unlink, stat, linesep, close
@@ -49,13 +49,12 @@ class RepositoryManager(object):
self.filter = [] if filter is None else filter
self.search = [] if search is None else search
self.timeout = timeout or 3
- self.factory = RepositoryFactory()
debug(u"Repository timeout setted to %ds" % self.timeout)
if cache_path is None:
self.cache_path = None
debug("No repository cache")
else:
- if not isfile(cache_path):
+ if not islocal(cache_path):
raise NotImplementedError("Repository cache must be local")
self.cache_path = abspath(cache_path)
# must_path is a list of directory which must exists
@@ -123,11 +122,11 @@ class RepositoryManager(object):
debug(u"Registering offline repository %s (%s)" % (config.path, config.name))
# we must force offline in cast of argument offline
config.offline = True
- self.repos.append(self.factory.create(config))
+ self.repos.append(RepositoryFactory(config))
# if path is local, no needs to create a cache
- elif isfile(config.path):
+ elif islocal(config.path):
debug(u"Registering direct repository %s (%s)" % (config.path, config.name))
- self.repos.append(self.factory.create(config))
+ self.repos.append(RepositoryFactory(config))
# path is remote, we need to create a cache
else:
debug(u"Registering cached repository %s (%s)" % (config.path, config.name))
@@ -194,7 +193,7 @@ class RepositoryManager(object):
# if something append bad during caching, we mark repo as offline
debug(u"Unable to cache repository %s: %s" % (config.name, e))
config.offline = True
- return self.factory.create(config)
+ return RepositoryFactory(config)
@property
def names(self):
@@ -241,7 +240,7 @@ class RepositoryManager(object):
raise ISError(u"No online repository")
ans = {}
for pattern in patterns:
- path, image, version = Repository.split_path(pattern)
+ path, image, version = split_path(pattern)
if image is None:
if path is None or version is None:
image = "*"
@@ -461,3 +460,103 @@ class RepositoryManager(object):
l.append(ln)
s = linesep.join(l)
out(s)
+
+class Database(object):
+ '''
+ Abstract repo database stuff
+ It needs to be local cause of sqlite3 which need to open a file
+ '''
+
+ version = 2.0
+
+ @classmethod
+ def create(cls, path):
+ arrow("Creating repository database")
+ # check locality
+ if not istools.isfile(path):
+ raise ISError("Database creation must be local")
+ path = os.path.abspath(path)
+ if os.path.exists(path):
+ raise ISError("Database already exists. Remove it before")
+ try:
+ conn = sqlite3.connect(path, isolation_level=None)
+ conn.execute("PRAGMA foreign_keys = ON")
+ conn.executescript(TEMPLATE_EMPTY_DB)
+ conn.execute("INSERT INTO repository values (?,?,?)",
+ (str(uuid.uuid4()), Database.version, "",))
+ conn.commit()
+ conn.close()
+ except Exception as e:
+ raise ISError(u"Create database failed", e)
+ return cls(path)
+
+ def __init__(self, path):
+ # check locality
+ if not istools.isfile(path):
+ raise ISError("Database must be local")
+ self.path = os.path.abspath(path)
+ if not os.path.exists(self.path):
+ raise ISError("Database not exists")
+ self.conn = sqlite3.connect(self.path, isolation_level=None)
+ self.conn.execute("PRAGMA foreign_keys = ON")
+ # get database version
+ try:
+ r = self.ask("SELECT version FROM repository").fetchone()
+ if r is None:
+ raise TypeError()
+ self.version = float(r[0])
+ except:
+ self.version = 1.0
+ if math.floor(self.version) >= math.floor(Database.version) + 1.0:
+ raise ISWarning(u"New database format (%s), please upgrade "
+ "your Installsystems version" % self.version)
+ # we make a query to be sure format is valid
+ try:
+ self.ask("SELECT * FROM image")
+ except:
+ debug(u"Invalid database format: %s" % self.version)
+ raise ISError("Invalid database format")
+
+ def begin(self):
+ '''
+ Start a db transaction
+ '''
+ self.conn.execute("BEGIN TRANSACTION")
+
+ def commit(self):
+ '''
+ Commit current db transaction
+ '''
+ self.conn.execute("COMMIT TRANSACTION")
+
+
+ def ask(self, sql, args=()):
+ '''
+ Ask question to db
+ '''
+ return self.conn.execute(sql, args)
+
+
+TEMPLATE_EMPTY_DB = u"""
+CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
+ name TEXT NOT NULL,
+ version TEXT NOT NULL,
+ date INTEGER NOT NULL,
+ author TEXT,
+ description TEXT,
+ size INTEGER NOT NULL,
+ is_min_version INTEGER NOT NULL,
+ format INTEGER NOT NULL,
+ UNIQUE(name, version));
+
+CREATE TABLE payload (md5 TEXT NOT NULL,
+ image_md5 TEXT NOT NULL REFERENCES image(md5),
+ name TEXT NOT NULL,
+ isdir INTEGER NOT NULL,
+ size INTEGER NOT NULL,
+ PRIMARY KEY(md5, image_md5));
+
+CREATE TABLE repository (uuid TEXT NOT NULL PRIMARY KEY,
+ version FLOAT NOT NULL,
+ motd TEXT NOT NULL);
+"""
diff --git a/installsystems/repository/repository.py b/installsystems/repository/repository.py
deleted file mode 100644
index 8167e1d3abda1549ae2bf65dfa296523f8b4848c..0000000000000000000000000000000000000000
--- a/installsystems/repository/repository.py
+++ /dev/null
@@ -1,521 +0,0 @@
-# -*- python -*-
-# -*- coding: utf-8 -*-
-
-# This file is part of Installsystems.
-#
-# Installsystems is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Installsystems is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Installsystems. If not, see .
-
-'''
-Abstract repository module
-'''
-
-from cStringIO import StringIO
-from installsystems.exception import ISError
-from installsystems.image.package import PackageImage
-from installsystems.printer import arrow, arrowlevel, out, warn, confirm
-from installsystems.repository.database import Database
-from installsystems.tools import isfile, chrights, mkdir, compare_versions, PipeFile
-from os import unlink, listdir, linesep, rmdir
-from os.path import join
-from os.path import join, basename, exists, isdir
-from re import match, split
-from time import time
-
-class Repository(object):
- '''
- Repository class
- '''
-
- @staticmethod
- def is_name(name):
- '''Check if name is a valid repository name'''
- return match("^[-_\w]+$", name) is not None
-
- @staticmethod
- def check_name(name):
- '''
- Raise exception is repository name is invalid
- '''
- if not Repository.is_name(name):
- raise ISError(u"Invalid repository name %s" % name)
- return name
-
- @staticmethod
- def split_path(path):
- '''
- Split an image path (repo/image:version)
- in a tuple (repo, image, version)
- '''
- x = match(u"^(?:([^/:]+)/)?([^/:]+)?(?::v?([^/:]+)?)?$", path)
- if x is None:
- raise ISError(u"invalid image path: %s" % path)
- return x.group(1, 2, 3)
-
- @staticmethod
- def split_list(repolist, filter=None):
- '''
- Return a list of repository from a comma/spaces separated names of repo
- '''
- if filter is None:
- filter = Repository.is_name
- return [r for r in split("[ ,\n\t\v]+", repolist) if filter(r)]
-
- @staticmethod
- def diff(repo1, repo2):
- '''
- Compute a diff between two repositories
- '''
- arrow(u"Diff between repositories #y#%s#R# and #g#%s#R#" % (repo1.config.name,
- repo2.config.name))
- # Get info from databases
- i_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
- "SELECT md5, name, version FROM image").fetchall())
- i_set1 = set(i_dict1.keys())
- i_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
- "SELECT md5, name, version FROM image").fetchall())
- i_set2 = set(i_dict2.keys())
- p_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
- "SELECT md5, name FROM payload").fetchall())
- p_set1 = set(p_dict1.keys())
- p_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
- "SELECT md5, name FROM payload").fetchall())
- p_set2 = set(p_dict2.keys())
- # computing diff
- i_only1 = i_set1 - i_set2
- i_only2 = i_set2 - i_set1
- p_only1 = p_set1 - p_set2
- p_only2 = p_set2 - p_set1
- # printing functions
- pimg = lambda r,c,m,d,: out("#%s#Image only in repository %s: %s v%s (%s)#R#" %
- (c, r.config.name, d[m][0], d[m][1], m))
- ppay = lambda r,c,m,d,: out("#%s#Payload only in repository %s: %s (%s)#R#" %
- (c, r.config.name, d[m][0], m))
- # printing image diff
- for md5 in i_only1: pimg(repo1, "y", md5, i_dict1)
- for md5 in p_only1: ppay(repo1, "y", md5, p_dict1)
- for md5 in i_only2: pimg(repo2, "g", md5, i_dict2)
- for md5 in p_only2: ppay(repo2, "g", md5, p_dict2)
-
- def __init__(self, config, db=None):
- self.config = config
- self.local = isfile(self.config.path)
- self.db = db
-
- def __getattribute__(self, name):
- '''
- Raise an error if repository is unavailable
- Unavailable can be caused because db is not accessible or
- because repository is not initialized
- '''
- config = object.__getattribute__(self, "config")
- # config, init, local and upgrade are always accessible
- if name in ("init", "config", "local", "upgrade"):
- return object.__getattribute__(self, name)
- # if no db (not init or not accessible) raise error
- if config.offline:
- raise ISError(u"Repository %s is offline" % config.name)
- return object.__getattribute__(self, name)
-
- @property
- def version(self):
- '''
- Return repository version
- '''
- raise NotImplementedError()
-
- @property
- def uuid(self):
- '''
- Return repository UUID
- '''
- return self.db.ask("SELECT uuid from repository").fetchone()[0]
-
- def init(self):
- '''
- Initialize an empty base repository
- '''
- config = self.config
- # check local repository
- if not self.local:
- raise ISError(u"Repository creation must be local")
- # create base directories
- arrow("Creating base directories")
- arrowlevel(1)
- # creating local directory
- try:
- if exists(config.path):
- arrow(u"%s already exists" % config.path)
- else:
- mkdir(config.path, config.uid, config.gid, config.dmod)
- arrow(u"%s directory created" % config.path)
- except Exception as e:
- raise ISError(u"Unable to create directory %s" % config.path, e)
- arrowlevel(-1)
- # create database
- d = Database.create(config.dbpath)
- chrights(config.dbpath, uid=config.uid,
- gid=config.gid, mode=config.fmod)
- # load database
- self.db = Database(config.dbpath)
- # mark repo as not offline
- self.config.offline = False
- # create/update last file
- self.update_last()
-
- def update_last(self):
- '''
- Update last file to current time
- '''
- # check local repository
- if not self.local:
- raise ISError(u"Repository must be local")
- try:
- arrow("Updating last file")
- last_path = join(self.config.path, self.config.lastname)
- open(last_path, "w").write("%s\n" % int(time()))
- chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
- except Exception as e:
- raise ISError(u"Update last file failed", e)
-
- def last(self, name):
- '''
- Return last version of name in repo or None if not found
- '''
- r = self.db.ask("SELECT version FROM image WHERE name = ?", (name,)).fetchall()
- # no row => no way
- if r is None:
- return None
- f = lambda x,y: x[0] if compare_versions(x[0], y[0]) > 0 else y[0]
- # return last
- return reduce(f, r)
-
- def _add(self, image):
- '''
- Add description to db
- '''
- arrow("Adding metadata")
- self.db.begin()
- # insert image information
- arrow("Image", 1)
- self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?,?,?)",
- (image.md5,
- image.name,
- image.version,
- image.date,
- image.author,
- image.description,
- image.size,
- image.is_min_version,
- image.format,
- ))
- # insert data information
- arrow("Payloads", 1)
- for name, obj in image.payload.items():
- self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
- (obj.md5,
- image.md5,
- name,
- obj.isdir,
- obj.size,
- ))
- # on commit
- self.db.commit()
- # update last file
- self.update_last()
-
- def add(self, image, delete=False):
- '''
- Add a packaged image to repository
- if delete is true, remove original files
- '''
- # check local repository
- if not self.local:
- raise ISError(u"Repository addition must be local")
- # cannot add already existant image
- if self.has(image.name, image.version):
- raise ISError(u"Image already in database, delete first!")
- # adding file to repository
- arrow("Copying images and payload")
- for obj in [ image ] + image.payload.values():
- dest = join(self.config.path, obj.md5)
- basesrc = basename(obj.path)
- if exists(dest):
- arrow(u"Skipping %s: already exists" % basesrc, 1)
- else:
- arrow(u"Adding %s (%s)" % (basesrc, obj.md5), 1)
- dfo = open(dest, "wb")
- sfo = PipeFile(obj.path, "r", progressbar=True)
- sfo.consume(dfo)
- sfo.close()
- dfo.close()
- chrights(dest, self.config.uid,
- self.config.gid, self.config.fmod)
- # copy is done. create a image inside repo
- r_image = PackageImage(join(self.config.path, image.md5),
- md5name=True)
- # checking must be done with original md5
- r_image.md5 = image.md5
- # checking image and payload after copy
- r_image.check("Check image and payload")
- self._add(image)
- # removing orginal files
- if delete:
- arrow("Removing original files")
- for obj in [ image ] + image.payload.values():
- arrow(basename(obj.path), 1)
- unlink(obj.path)
-
- def getallmd5(self):
- '''
- Get list of all md5 in DB
- '''
- res = self.db.ask("SELECT md5 FROM image UNION SELECT md5 FROM payload").fetchall()
- return [ md5[0] for md5 in res ]
-
- def check(self):
- '''
- Check repository for unreferenced and missing files
- '''
- # Check if the repo is local
- if not self.local:
- raise ISError(u"Repository must be local")
- local_files = set(listdir(self.config.path))
- local_files.remove(self.config.dbname)
- local_files.remove(self.config.lastname)
- db_files = set(self.getallmd5())
- # check missing files
- arrow("Checking missing files")
- missing_files = db_files - local_files
- if len(missing_files) > 0:
- out(linesep.join(missing_files))
- # check unreferenced files
- arrow("Checking unreferenced files")
- unref_files = local_files - db_files
- if len(unref_files) > 0:
- out(linesep.join(unref_files))
- # check corruption of local files
- arrow("Checking corrupted files")
- for f in local_files:
- fo = PipeFile(join(self.config.path, f))
- fo.consume()
- fo.close()
- if fo.md5 != f:
- out(f)
-
- def clean(self, force=False):
- '''
- Clean the repository's content
- '''
- # Check if the repo is local
- if not self.local:
- raise ISError(u"Repository must be local")
- allmd5 = set(self.getallmd5())
- repofiles = set(listdir(self.config.path)) - set([self.config.dbname, self.config.lastname])
- dirtyfiles = repofiles - allmd5
- if len(dirtyfiles) > 0:
- # print dirty files
- arrow("Dirty files:")
- for f in dirtyfiles:
- arrow(f, 1)
- # ask confirmation
- if not force and not confirm("Remove dirty files? (yes) "):
- raise ISError(u"Aborted!")
- # start cleaning
- arrow("Cleaning")
- for f in dirtyfiles:
- p = join(self.config.path, f)
- arrow(u"Removing %s" % p, 1)
- try:
- if isdir(p):
- rmdir(p)
- else:
- unlink(p)
- except:
- warn(u"Removing %s failed" % p)
- else:
- arrow("Nothing to clean")
-
- def delete(self, name, version, payloads=True):
- '''
- Delete an image from repository
- '''
- # check local repository
- if not self.local:
- raise ISError(u"Repository deletion must be local")
- # get md5 of files related to images (exception is raised if not exists
- md5s = self.getmd5(name, version)
- # cleaning db (must be done before cleaning)
- arrow("Cleaning database")
- arrow("Remove payloads from database", 1)
- self.db.begin()
- for md5 in md5s[1:]:
- self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?",
- (md5, md5s[0])).fetchone()
- arrow("Remove image from database", 1)
- self.db.ask("DELETE FROM image WHERE md5 = ?",
- (md5s[0],)).fetchone()
- self.db.commit()
- # Removing files
- arrow("Removing files from pool")
- # if asked don't remove payloads
- if not payloads:
- md5s = [ md5s[0] ]
- arrowlevel(1)
- for md5 in md5s:
- self._remove_file(md5)
- arrowlevel(-1)
- # update last file
- self.update_last()
-
- def images(self):
- '''
- Return a dict of information on images
- '''
- db_images = self.db.ask("SELECT md5, name, version, date, author, \
- description, size, is_min_version, format \
- FROM image ORDER BY name, version").fetchall()
-
- images = []
- field = ("md5", "name", "version", "date", "author", "description",
- "size", "is_min_version", "format")
- for info in db_images:
- d = dict(zip(field, info))
- d["repo"] = self.config.name
- d["url"] = join(self.config.path, d["md5"])
- images.append(d)
- return images
-
- def payloads(self):
- '''
- Return a dict of information on payloads
- '''
- db_payloads = self.db.ask("SELECT payload.md5,payload.size,payload.isdir,image.name,image.version,payload.name FROM payload inner join image on payload.image_md5 = image.md5").fetchall()
- res = {}
- for payload in db_payloads:
- md5 = payload[0]
- # create entry if not exists
- if md5 not in res:
- res[md5] = {"size": payload[1], "isdir": payload[2], "images": {}}
- # add image to list
- imgpath = u"%s/%s:%s" % (self.config.name, payload[3], payload[4])
- res[md5]["images"][imgpath] = {"repo": self.config.name,
- "imgname": payload[3],
- "imgver": payload[4],
- "payname": payload[5]}
- return res
-
- def search(self, pattern):
- '''
- Search pattern in a repository
- '''
- images = self.db.ask("SELECT name, version, author, description\
- FROM image\
- WHERE name LIKE ? OR\
- description LIKE ? OR\
- author LIKE ?",
- tuple( [u"%%%s%%" % pattern ] * 3)
- ).fetchall()
- for name, version, author, description in images:
- arrow(u"%s v%s" % (name, version), 1)
- out(u" #yellow#Author:#reset# %s" % author)
- out(u" #yellow#Description:#reset# %s" % description)
-
- def _remove_file(self, filename):
- '''
- Remove a filename from pool. Check if it's not needed by db before
- '''
- # check existance in table image
- have = False
- for table in ("image", "payload"):
- have = have or self.db.ask(u"SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table,
- (filename,)).fetchone() is not None
- # if no reference, delete!
- if not have:
- arrow(u"%s, deleted" % filename)
- unlink(join(self.config.path, filename))
- else:
- arrow(u"%s, skipped" % filename)
-
- def has(self, name, version):
- '''
- Return the existance of a package
- '''
- return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None
-
- def get(self, name, version=None):
- '''
- Return an image from a name and version
- '''
- # is no version take the last
- if version is None:
- version = self.last(name)
- if version is None:
- raise ISError(u"Unable to find image %s in %s" % (name,
- self.config.name))
- # get file md5 from db
- r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
- (name, version)).fetchone()
- if r is None:
- raise ISError(u"Unable to find image %s v%s in %s" % (name, version,
- self.config.name))
- path = join(self.config.path, r[0])
- # getting the file
- arrow(u"Loading image %s v%s from repository %s" % (name,
- version,
- self.config.name))
- memfile = StringIO()
- try:
- fo = PipeFile(path, "r")
- fo.consume(memfile)
- fo.close()
- except Exception as e:
- raise ISError(u"Loading image %s v%s failed" % (name, version), e)
- memfile.seek(0)
- pkg = PackageImage(path, fileobj=memfile, md5name=True)
- if pkg.md5 != r[0]:
- raise ISError(u"Image MD5 verification failure")
- return pkg
-
- def getmd5(self, name, version):
- '''
- Return an image md5 and payload md5 from name and version. Order matter !
- Image md5 will still be the first
- '''
- # get file md5 from db
- a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1",
- (name,version)).fetchone()
- if a is None:
- raise ISError(u"No such image %s version %s" % (name, version))
- b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?",
- (a[0],)).fetchall()
- return [ a[0] ] + [ x[0] for x in b ]
-
- @property
- def motd(self):
- '''
- Return repository message of the day
- '''
- motd = self.db.ask("SELECT motd FROM repository").fetchone()[0]
- return None if len(motd) == 0 else motd
-
- def setmotd(self, value=""):
- '''
- Set repository message of the day
- '''
- # check local repository
- if not self.local:
- raise ISError(u"Repository must be local")
- arrow("Updating motd")
- self.db.ask("UPDATE repository SET motd = ?", (value,))
- self.update_last()
diff --git a/installsystems/repository/repository1.py b/installsystems/repository/repository1.py
index 82b3b907397186473398f3f0e5ac494a9e9cb701..17839ef2c853c2034e0cbcc205eebd5e0c6971eb 100644
--- a/installsystems/repository/repository1.py
+++ b/installsystems/repository/repository1.py
@@ -17,25 +17,144 @@
# along with Installsystems. If not, see .
'''
-Repository v1
+Repository stuff
'''
-from installsystems.image.package import PackageImage
-from installsystems.printer import arrow, arrowlevel, warn, info
-from installsystems.repository.config import RepositoryConfig
-from installsystems.repository.database import Database
-from installsystems.repository.repository import Repository
-from os import listdir, unlink, symlink
-from os.path import join, exists
-from shutil import move, rmtree
-from tempfile import mkdtemp
+from installsystems.tools import islocal
-class Repository1(Repository):
+class Repository1(object):
+ '''
+ Repository class
+ '''
- def _add(self, image):
+ def __init__(self, config):
+ self.config = config
+ self.local = islocal(config.path)
+ if not self.config.offline:
+ try:
+ self.db = Database(config.dbpath)
+ except:
+ debug(u"Unable to load database %s" % config.dbpath)
+ self.config.offline = True
+ if self.config.offline:
+ debug(u"Repository %s is offline" % config.name)
+
+ def __getattribute__(self, name):
+ '''
+ Raise an error if repository is unavailable
+ Unavailable can be caused because db is not accessible or
+ because repository is not initialized
'''
- Add description to db
+ config = object.__getattribute__(self, "config")
+ # config, init, local are always accessible
+ if name in ("init", "config", "local"):
+ return object.__getattribute__(self, name)
+ # if no db (not init or not accessible) raise error
+ if config.offline:
+ raise ISError(u"Repository %s is offline" % config.name)
+ return object.__getattribute__(self, name)
+
+ @property
+ def version(self):
+ '''
+ Return repository version
+ '''
+ return self.db.version
+
+ def init(self):
'''
+ Initialize an empty base repository
+ '''
+ config = self.config
+ # check local repository
+ if not self.local:
+ raise ISError(u"Repository creation must be local")
+ # create base directories
+ arrow("Creating base directories")
+ arrowlevel(1)
+ # creating local directory
+ try:
+ if os.path.exists(config.path):
+ arrow(u"%s already exists" % config.path)
+ else:
+ istools.mkdir(config.path, config.uid, config.gid, config.dmod)
+ arrow(u"%s directory created" % config.path)
+ except Exception as e:
+ raise ISError(u"Unable to create directory %s" % config.path, e)
+ arrowlevel(-1)
+ # create database
+ d = Database.create(config.dbpath)
+ istools.chrights(config.dbpath, uid=config.uid,
+ gid=config.gid, mode=config.fmod)
+ # load database
+ self.db = Database(config.dbpath)
+ # mark repo as not offline
+ self.config.offline = False
+ # create/update last file
+ self.update_last()
+
+ def update_last(self):
+ '''
+ Update last file to current time
+ '''
+ # check local repository
+ if not self.local:
+ raise ISError(u"Repository addition must be local")
+ try:
+ arrow("Updating last file")
+ last_path = os.path.join(self.config.path, self.config.lastname)
+ open(last_path, "w").write("%s\n" % int(time.time()))
+ istools.chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
+ except Exception as e:
+ raise ISError(u"Update last file failed", e)
+
+ def last(self, name):
+ '''
+ Return last version of name in repo or None if not found
+ '''
+ r = self.db.ask("SELECT version FROM image WHERE name = ?", (name,)).fetchall()
+ # no row => no way
+ if r is None:
+ return None
+ f = lambda x,y: x[0] if istools.compare_versions(x[0], y[0]) > 0 else y[0]
+ # return last
+ return reduce(f, r)
+
+ def add(self, image, delete=False):
+ '''
+ Add a packaged image to repository
+ if delete is true, remove original files
+ '''
+ # check local repository
+ if not self.local:
+ raise ISError(u"Repository addition must be local")
+ # cannot add already existant image
+ if self.has(image.name, image.version):
+ raise ISError(u"Image already in database, delete first!")
+ # adding file to repository
+ arrow("Copying images and payload")
+ for obj in [ image ] + image.payload.values():
+ dest = os.path.join(self.config.path, obj.md5)
+ basesrc = os.path.basename(obj.path)
+ if os.path.exists(dest):
+ arrow(u"Skipping %s: already exists" % basesrc, 1)
+ else:
+ arrow(u"Adding %s (%s)" % (basesrc, obj.md5), 1)
+ dfo = open(dest, "wb")
+ sfo = PipeFile(obj.path, "r", progressbar=True)
+ sfo.consume(dfo)
+ sfo.close()
+ dfo.close()
+ istools.chrights(dest, self.config.uid,
+ self.config.gid, self.config.fmod)
+ # copy is done. create a image inside repo
+ r_image = PackageImage(os.path.join(self.config.path, image.md5),
+ md5name=True)
+ # checking must be done with original md5
+ r_image.md5 = image.md5
+ # checking image and payload after copy
+ r_image.check("Check image and payload")
+ # add description to db
arrow("Adding metadata")
self.db.begin()
# insert image information
@@ -49,7 +168,7 @@ class Repository1(Repository):
image.description,
image.size,
))
- # insert data information
+ # insert data informations
arrow("Payloads", 1)
for name, obj in image.payload.items():
self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
@@ -63,103 +182,298 @@ class Repository1(Repository):
self.db.commit()
# update last file
self.update_last()
+ # removing orginal files
+ if delete:
+ arrow("Removing original files")
+ for obj in [ image ] + image.payload.values():
+ arrow(os.path.basename(obj.path), 1)
+ os.unlink(obj.path)
+
+ def getallmd5(self):
+ '''
+ Get list of all md5 in DB
+ '''
+ res = self.db.ask("SELECT md5 FROM image UNION SELECT md5 FROM payload").fetchall()
+ return [ md5[0] for md5 in res ]
+
+ def check(self):
+ '''
+ Check repository for unreferenced and missing files
+ '''
+ # Check if the repo is local
+ if not self.local:
+ raise ISError(u"Repository must be local")
+ local_files = set(os.listdir(self.config.path))
+ local_files.remove(self.config.dbname)
+ local_files.remove(self.config.lastname)
+ db_files = set(self.getallmd5())
+ # check missing files
+ arrow("Checking missing files")
+ missing_files = db_files - local_files
+ if len(missing_files) > 0:
+ out(os.linesep.join(missing_files))
+ # check unreferenced files
+ arrow("Checking unreferenced files")
+ unref_files = local_files - db_files
+ if len(unref_files) > 0:
+ out(os.linesep.join(unref_files))
+ # check corruption of local files
+ arrow("Checking corrupted files")
+ for f in local_files:
+ fo = PipeFile(os.path.join(self.config.path, f))
+ fo.consume()
+ fo.close()
+ if fo.md5 != f:
+ out(f)
+
+ def clean(self, force=False):
+ '''
+ Clean the repository's content
+ '''
+ # Check if the repo is local
+ if not self.local:
+ raise ISError(u"Repository must be local")
+ allmd5 = set(self.getallmd5())
+ repofiles = set(os.listdir(self.config.path)) - set([self.config.dbname, self.config.lastname])
+ dirtyfiles = repofiles - allmd5
+ if len(dirtyfiles) > 0:
+ # print dirty files
+ arrow("Dirty files:")
+ for f in dirtyfiles:
+ arrow(f, 1)
+ # ask confirmation
+ if not force and not confirm("Remove dirty files? (yes) "):
+ raise ISError(u"Aborted!")
+ # start cleaning
+ arrow("Cleaning")
+ for f in dirtyfiles:
+ p = os.path.join(self.config.path, f)
+ arrow(u"Removing %s" % p, 1)
+ try:
+ if os.path.isdir(p):
+ os.rmdir(p)
+ else:
+ os.unlink(p)
+ except:
+ warn(u"Removing %s failed" % p)
+ else:
+ arrow("Nothing to clean")
+
+ def delete(self, name, version, payloads=True):
+ '''
+ Delete an image from repository
+ '''
+ # check local repository
+ if not self.local:
+ raise ISError(u"Repository deletion must be local")
+ # get md5 of files related to images (exception is raised if not exists
+ md5s = self.getmd5(name, version)
+ # cleaning db (must be done before cleaning)
+ arrow("Cleaning database")
+ arrow("Remove payloads from database", 1)
+ self.db.begin()
+ for md5 in md5s[1:]:
+ self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?",
+ (md5, md5s[0])).fetchone()
+ arrow("Remove image from database", 1)
+ self.db.ask("DELETE FROM image WHERE md5 = ?",
+ (md5s[0],)).fetchone()
+ self.db.commit()
+ # Removing files
+ arrow("Removing files from pool")
+ # if asked don't remove payloads
+ if not payloads:
+ md5s = [ md5s[0] ]
+ arrowlevel(1)
+ for md5 in md5s:
+ self._remove_file(md5)
+ arrowlevel(-1)
+ # update last file
+ self.update_last()
def images(self):
'''
Return a dict of information on images
'''
- db_images = self.db.ask("SELECT md5, name, version, date, author, \
- description, size \
- FROM image ORDER BY name, version").fetchall()
-
+ db_images = self.db.ask("SELECT md5, name, version, date,\
+ author, description, size FROM image ORDER BY name, version").fetchall()
images = []
- field = ("md5", "name", "version", "date", "author", "description",
- "size")
+ field = ("md5", "name", "version", "date", "author", "description", "size")
for info in db_images:
d = dict(zip(field, info))
d["repo"] = self.config.name
- d["url"] = join(self.config.path, d["md5"])
- d["format"] = 1
- d["is_min_version"] = 9
+ d["url"] = os.path.join(self.config.path, d["md5"])
images.append(d)
return images
- @property
- def uuid(self):
+ def payloads(self):
'''
- Repository v1 doesn't support UUID
+ Return a dict of information on payloads
'''
- return None
+ db_payloads = self.db.ask("SELECT payload.md5,payload.size,payload.isdir,image.name,image.version,payload.name FROM payload inner join image on payload.image_md5 = image.md5").fetchall()
+ res = {}
+ for payload in db_payloads:
+ md5 = payload[0]
+ # create entry if not exists
+ if md5 not in res:
+ res[md5] = {"size": payload[1], "isdir": payload[2], "images": {}}
+ # add image to list
+ imgpath = u"%s/%s:%s" % (self.config.name, payload[3], payload[4])
+ res[md5]["images"][imgpath] = {"repo": self.config.name,
+ "imgname": payload[3],
+ "imgver": payload[4],
+ "payname": payload[5]}
+ return res
- @property
- def motd(self):
+ def search(self, pattern):
'''
- Return repository message of the day.
- Repository v1 don't have message of day
+ Search pattern in a repository
'''
- return None
+ images = self.db.ask("SELECT name, version, author, description\
+ FROM image\
+ WHERE name LIKE ? OR\
+ description LIKE ? OR\
+ author LIKE ?",
+ tuple( [u"%%%s%%" % pattern ] * 3)
+ ).fetchall()
+ for name, version, author, description in images:
+ arrow(u"%s v%s" % (name, version), 1)
+ out(u" #yellow#Author:#reset# %s" % author)
+ out(u" #yellow#Description:#reset# %s" % description)
- def setmotd(self, value=""):
+ def _remove_file(self, filename):
'''
- Don't set repository message of the day. Not supported by v1.
+ Remove a filename from pool. Check if it's not needed by db before
'''
- # check local repository
- warn(u"Repository v1 doesn't support motd. Unable to set")
+ # check existance in table image
+ have = False
+ for table in ("image", "payload"):
+ have = have or self.db.ask(u"SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table,
+ (filename,)).fetchone() is not None
+ # if no reference, delete!
+ if not have:
+ arrow(u"%s, deleted" % filename)
+ os.unlink(os.path.join(self.config.path, filename))
+ else:
+ arrow(u"%s, skipped" % filename)
- @property
- def version(self):
+ def has(self, name, version):
'''
- Return repository version
+ Return the existance of a package
+ '''
+ return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None
+
+ def get(self, name, version=None):
+ '''
+ Return an image from a name and version
+ '''
+ # is no version take the last
+ if version is None:
+ version = self.last(name)
+ if version is None:
+ raise ISError(u"Unable to find image %s in %s" % (name,
+ self.config.name))
+ # get file md5 from db
+ r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
+ (name, version)).fetchone()
+ if r is None:
+ raise ISError(u"Unable to find image %s v%s in %s" % (name, version,
+ self.config.name))
+ path = os.path.join(self.config.path, r[0])
+ # getting the file
+ arrow(u"Loading image %s v%s from repository %s" % (name,
+ version,
+ self.config.name))
+ memfile = cStringIO.StringIO()
+ try:
+ fo = PipeFile(path, "r")
+ fo.consume(memfile)
+ fo.close()
+ except Exception as e:
+ raise ISError(u"Loading image %s v%s failed" % (name, version), e)
+ memfile.seek(0)
+ pkg = PackageImage(path, fileobj=memfile, md5name=True)
+ if pkg.md5 != r[0]:
+ raise ISError(u"Image MD5 verification failure")
+ return pkg
+
+ def getmd5(self, name, version):
+ '''
+ Return an image md5 and payload md5 from name and version. Order matter !
+ Image md5 will still be the first
+ '''
+ # get file md5 from db
+ a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1",
+ (name,version)).fetchone()
+ if a is None:
+ raise ISError(u"No such image %s version %s" % (name, version))
+ b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?",
+ (a[0],)).fetchall()
+ return [ a[0] ] + [ x[0] for x in b ]
+
+
+ @classmethod
+ def create(cls, path):
+ arrow("Creating repository database")
+ # check locality
+ if not istools.isfile(path):
+ raise ISError("Database creation must be local")
+ path = os.path.abspath(path)
+ if os.path.exists(path):
+ raise ISError("Database already exists. Remove it before")
+ try:
+ conn = sqlite3.connect(path, isolation_level=None)
+ conn.execute("PRAGMA foreign_keys = ON")
+ conn.executescript(istemplate.createdb)
+ conn.commit()
+ conn.close()
+ except Exception as e:
+ raise ISError(u"Create database failed", e)
+ return cls(path)
+
+ def __init__(self, path):
+ # check locality
+ if not istools.isfile(path):
+ raise ISError("Database must be local")
+ self.path = os.path.abspath(path)
+ if not os.path.exists(self.path):
+ raise ISError("Database not exists")
+ self.conn = sqlite3.connect(self.path, isolation_level=None)
+ self.conn.execute("PRAGMA foreign_keys = ON")
+ # get database version
+ try:
+ r = self.ask("SELECT value FROM misc WHERE key = 'version'").fetchone()
+ if r is None:
+ raise TypeError()
+ self.version = float(r[0])
+ except:
+ self.version = 1.0
+ # we only support database v1
+ if self.version >= 2.0:
+ debug(u"Invalid database format: %s" % self.version)
+ raise ISError("Invalid database format")
+ # we make a query to be sure format is valid
+ try:
+ self.ask("SELECT * FROM image")
+ except:
+ debug(u"Invalid database format: %s" % self.version)
+ raise ISError("Invalid database format")
+
+ def begin(self):
+ '''
+ Start a db transaction
+ '''
+ self.conn.execute("BEGIN TRANSACTION")
+
+ def commit(self):
+ '''
+ Commit current db transaction
+ '''
+ self.conn.execute("COMMIT TRANSACTION")
+
+
+ def ask(self, sql, args=()):
+ '''
+ Ask question to db
'''
- return 1
-
- def upgrade(self):
- raise NotImplementedError()
- # if self.version == Database.version:
- # info("Repository already up-to-date (%s)" % self.version)
- # return
- # else:
- # arrow("Start repository upgrade")
- # arrowlevel(1)
- # # Create dummy repository
- # tmpdir = mkdtemp()
- # try:
- # repoconf = RepositoryConfig("tmp_migrate_repo", path=tmpdir)
- # dstrepo = Repository(repoconf)
- # # Symlink content from repository into dummy repo
- # for file in listdir(self.config.path):
- # symlink(join(self.config.path, file),
- # join(tmpdir, file))
- # unlink(repoconf.dbpath)
- # unlink(repoconf.lastpath)
- # old_verbosity = installsystems.verbosity
- # arrow("Initialize new database")
- # # Disable unwanted message during upgrade
- # installsystems.verbosity = 0
- # dstrepo.init()
- # # Restore verbosity
- # installsystems.verbosity = old_verbosity
- # md5s = self.db.ask("SELECT md5 FROM image").fetchall()
- # # Copy images to dummy repository (fill new database)
- # arrow("Fill database with images")
- # arrowlevel(1)
- # installsystems.verbosity = 0
- # for img in [PackageImage(join(self.config.path, md5[0]),
- # md5name=True) for md5 in md5s]:
- # installsystems.verbosity = old_verbosity
- # arrow("%s v%s" % (img.name, img.version))
- # installsystems.verbosity = 0
- # dstrepo.add(img)
- # installsystems.verbosity = old_verbosity
- # arrowlevel(-1)
- # arrow("Backup old database")
- # move(self.config.dbpath,
- # join("%s.bak" % self.config.dbpath))
- # # Replace old db with the new from dummy repository
- # move(repoconf.dbpath, self.config.dbpath)
- # self.update_last()
- # arrowlevel(-1)
- # arrow("Repository upgrade complete")
- # finally:
- # # Remove dummy repository
- # rmtree(tmpdir)
+ return self.conn.execute(sql, args)
diff --git a/installsystems/repository/repository2.py b/installsystems/repository/repository2.py
index 1661015de497492494aef486a5011b134677067a..ae45f2948354d0ddcd3829e955fb69284301cc32 100644
--- a/installsystems/repository/repository2.py
+++ b/installsystems/repository/repository2.py
@@ -26,7 +26,7 @@ from installsystems.image.package import PackageImage
from installsystems.printer import arrow, arrowlevel, warn, info, out, confirm
from installsystems.repository.database import Database
from installsystems.repository.repository1 import Repository1
-from installsystems.tools import PipeFile, isfile, chrights, mkdir, compare_versions
+from installsystems.tools import PipeFile, islocal, chrights, mkdir, compare_versions
from os import unlink, listdir, linesep, rmdir, symlink
from os.path import join, exists, basename, isdir
from shutil import move, rmtree
@@ -36,6 +36,285 @@ class Repository2(Repository1):
Repository class
'''
+ def __init__(self, config, db=None):
+ self.config = config
+ self.local = islocal(self.config.path)
+ self.db = db
+
+ def __getattribute__(self, name):
+ '''
+ Raise an error if repository is unavailable
+ Unavailable can be caused because db is not accessible or
+ because repository is not initialized
+ '''
+ config = object.__getattribute__(self, "config")
+ # config, init, local and upgrade are always accessible
+ if name in ("init", "config", "local"):
+ return object.__getattribute__(self, name)
+ # if no db (not init or not accessible) raise error
+ if config.offline:
+ raise ISError(u"Repository %s is offline" % config.name)
+ return object.__getattribute__(self, name)
+
+ def update_last(self):
+ '''
+ Update last file to current time
+ '''
+ # check local repository
+ if not self.local:
+ raise ISError(u"Repository must be local")
+ try:
+ arrow("Updating last file")
+ last_path = join(self.config.path, self.config.lastname)
+ open(last_path, "w").write("%s\n" % int(time()))
+ chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
+ except Exception as e:
+ raise ISError(u"Update last file failed", e)
+
+ def last(self, name):
+ '''
+ Return last version of name in repo or None if not found
+ '''
+ r = self.db.ask("SELECT version FROM image WHERE name = ?", (name,)).fetchall()
+ # no row => no way
+ if r is None:
+ return None
+ f = lambda x,y: x[0] if compare_versions(x[0], y[0]) > 0 else y[0]
+ # return last
+ return reduce(f, r)
+
+ def add(self, image, delete=False):
+ '''
+ Add a packaged image to repository
+ if delete is true, remove source files
+ '''
+ raise NotImplementedError()
+
+ def getallmd5(self):
+ '''
+ Get list of all md5 in DB
+ '''
+ res = self.db.ask("SELECT md5 FROM image UNION SELECT md5 FROM payload").fetchall()
+ return [ md5[0] for md5 in res ]
+
+ def check(self):
+ '''
+ Check repository for unreferenced and missing files
+ '''
+ # Check if the repo is local
+ if not self.local:
+ raise ISError(u"Repository must be local")
+ local_files = set(listdir(self.config.path))
+ local_files.remove(self.config.dbname)
+ local_files.remove(self.config.lastname)
+ db_files = set(self.getallmd5())
+ # check missing files
+ arrow("Checking missing files")
+ missing_files = db_files - local_files
+ if len(missing_files) > 0:
+ out(linesep.join(missing_files))
+ # check unreferenced files
+ arrow("Checking unreferenced files")
+ unref_files = local_files - db_files
+ if len(unref_files) > 0:
+ out(linesep.join(unref_files))
+ # check corruption of local files
+ arrow("Checking corrupted files")
+ for f in local_files:
+ fo = PipeFile(join(self.config.path, f))
+ fo.consume()
+ fo.close()
+ if fo.md5 != f:
+ out(f)
+
+ def clean(self, force=False):
+ '''
+ Clean the repository's content
+ '''
+ # Check if the repo is local
+ if not self.local:
+ raise ISError(u"Repository must be local")
+ allmd5 = set(self.getallmd5())
+ repofiles = set(listdir(self.config.path)) - set([self.config.dbname, self.config.lastname])
+ dirtyfiles = repofiles - allmd5
+ if len(dirtyfiles) > 0:
+ # print dirty files
+ arrow("Dirty files:")
+ for f in dirtyfiles:
+ arrow(f, 1)
+ # ask confirmation
+ if not force and not confirm("Remove dirty files? (yes) "):
+ raise ISError(u"Aborted!")
+ # start cleaning
+ arrow("Cleaning")
+ for f in dirtyfiles:
+ p = join(self.config.path, f)
+ arrow(u"Removing %s" % p, 1)
+ try:
+ if isdir(p):
+ rmdir(p)
+ else:
+ unlink(p)
+ except:
+ warn(u"Removing %s failed" % p)
+ else:
+ arrow("Nothing to clean")
+
+ def delete(self, name, version, payloads=True):
+ '''
+ Delete an image from repository
+ '''
+ # check local repository
+ if not self.local:
+ raise ISError(u"Repository deletion must be local")
+ # get md5 of files related to images (exception is raised if not exists
+ md5s = self.getmd5(name, version)
+ # cleaning db (must be done before cleaning)
+ arrow("Cleaning database")
+ arrow("Remove payloads from database", 1)
+ self.db.begin()
+ for md5 in md5s[1:]:
+ self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?",
+ (md5, md5s[0])).fetchone()
+ arrow("Remove image from database", 1)
+ self.db.ask("DELETE FROM image WHERE md5 = ?",
+ (md5s[0],)).fetchone()
+ self.db.commit()
+ # Removing files
+ arrow("Removing files from pool")
+ # if asked don't remove payloads
+ if not payloads:
+ md5s = [ md5s[0] ]
+ arrowlevel(1)
+ for md5 in md5s:
+ self._remove_file(md5)
+ arrowlevel(-1)
+ # update last file
+ self.update_last()
+
+ def images(self):
+ '''
+ Return a dict of information on images
+ '''
+ db_images = self.db.ask("SELECT md5, name, version, date, author, \
+ description, size, is_min_version, format \
+ FROM image ORDER BY name, version").fetchall()
+
+ images = []
+ field = ("md5", "name", "version", "date", "author", "description",
+ "size", "is_min_version", "format")
+ for info in db_images:
+ d = dict(zip(field, info))
+ d["repo"] = self.config.name
+ d["url"] = join(self.config.path, d["md5"])
+ images.append(d)
+ return images
+
+ def payloads(self):
+ '''
+ Return a dict of information on payloads
+ '''
+ db_payloads = self.db.ask("SELECT payload.md5,payload.size,payload.isdir,image.name,image.version,payload.name FROM payload inner join image on payload.image_md5 = image.md5").fetchall()
+ res = {}
+ for payload in db_payloads:
+ md5 = payload[0]
+ # create entry if not exists
+ if md5 not in res:
+ res[md5] = {"size": payload[1], "isdir": payload[2], "images": {}}
+ # add image to list
+ imgpath = u"%s/%s:%s" % (self.config.name, payload[3], payload[4])
+ res[md5]["images"][imgpath] = {"repo": self.config.name,
+ "imgname": payload[3],
+ "imgver": payload[4],
+ "payname": payload[5]}
+ return res
+
+ def search(self, pattern):
+ '''
+ Search pattern in a repository
+ '''
+ images = self.db.ask("SELECT name, version, author, description\
+ FROM image\
+ WHERE name LIKE ? OR\
+ description LIKE ? OR\
+ author LIKE ?",
+ tuple( [u"%%%s%%" % pattern ] * 3)
+ ).fetchall()
+ for name, version, author, description in images:
+ arrow(u"%s v%s" % (name, version), 1)
+ out(u" #yellow#Author:#reset# %s" % author)
+ out(u" #yellow#Description:#reset# %s" % description)
+
+ def _remove_file(self, filename):
+ '''
+ Remove a filename from pool. Check if it's not needed by db before
+ '''
+ # check existance in table image
+ have = False
+ for table in ("image", "payload"):
+ have = have or self.db.ask(u"SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table,
+ (filename,)).fetchone() is not None
+ # if no reference, delete!
+ if not have:
+ arrow(u"%s, deleted" % filename)
+ unlink(join(self.config.path, filename))
+ else:
+ arrow(u"%s, skipped" % filename)
+
+ def has(self, name, version):
+ '''
+ Return the existance of a package
+ '''
+ return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None
+
+ def get(self, name, version=None):
+ '''
+ Return an image from a name and version
+ '''
+ # is no version take the last
+ if version is None:
+ version = self.last(name)
+ if version is None:
+ raise ISError(u"Unable to find image %s in %s" % (name,
+ self.config.name))
+ # get file md5 from db
+ r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
+ (name, version)).fetchone()
+ if r is None:
+ raise ISError(u"Unable to find image %s v%s in %s" % (name, version,
+ self.config.name))
+ path = join(self.config.path, r[0])
+ # getting the file
+ arrow(u"Loading image %s v%s from repository %s" % (name,
+ version,
+ self.config.name))
+ memfile = StringIO()
+ try:
+ fo = PipeFile(path, "r")
+ fo.consume(memfile)
+ fo.close()
+ except Exception as e:
+ raise ISError(u"Loading image %s v%s failed" % (name, version), e)
+ memfile.seek(0)
+ pkg = PackageImage(path, fileobj=memfile, md5name=True)
+ if pkg.md5 != r[0]:
+ raise ISError(u"Image MD5 verification failure")
+ return pkg
+
+ def getmd5(self, name, version):
+ '''
+ Return an image md5 and payload md5 from name and version. Order matter !
+ Image md5 will still be the first
+ '''
+ # get file md5 from db
+ a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1",
+ (name,version)).fetchone()
+ if a is None:
+ raise ISError(u"No such image %s version %s" % (name, version))
+ b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?",
+ (a[0],)).fetchall()
+ return [ a[0] ] + [ x[0] for x in b ]
+
@property
def version(self):
'''
@@ -416,4 +695,8 @@ class Repository2(Repository1):
self.update_last()
def upgrade(self):
+ '''
+ Upgrade database to the next version
+ '''
info("No upgrade available")
+