Commit 5a136f19 authored by Seblu's avatar Seblu

temp rework repo

parent 573805ed
......@@ -28,7 +28,9 @@ from installsystems.exception import ISError, ISException
from installsystems.image import PackageImage, SourceImage
from installsystems.printer import arrow, arrowlevel, setmode
from installsystems.printer import out, warn, error, debug, confirm
from installsystems.repository import Repository, RepositoryManager, RepositoryConfig
from installsystems.repository import split_list as split_repo_list, diff as repodiff
from installsystems.repository.manager import RepositoryManager
from installsystems.repository.config import RepositoryConfig
from installsystems.tools import chroot, prepare_chroot, unprepare_chroot
from installsystems.tools import islocal, smd5sum, argv
from os import getpid, getcwdu, chdir
......@@ -48,8 +50,8 @@ def load_repositories(args):
if args.no_cache:
args.cache = None
# split filter and search in list
args.repo_filter = Repository.split_list(args.repo_filter)
args.repo_search = Repository.split_list(args.repo_search)
args.repo_filter = split_repo_list(args.repo_filter)
args.repo_search = split_repo_list(args.repo_search)
# init repo cache object
repoman = RepositoryManager(args.cache, timeout=args.repo_timeout or args.timeout,
filter=args.repo_filter, search=args.repo_search)
......@@ -252,7 +254,7 @@ def c_diff(args):
repoman = load_repositories(args)
if args.object[0] in repoman.onlines and args.object[1] in repoman.onlines:
try:
Repository.diff(repoman[args.object[0]], repoman[args.object[1]])
diff(repoman[args.object[0]], repoman[args.object[1]])
except IndexError as e:
raise ISError(e)
else:
......
......@@ -34,7 +34,7 @@ from installsystems.tools import mkdir, abspath, time_rfc2822, human_size, argv,
from json import loads, dumps
from math import floor
from os import listdir
from os.path import join, basename, exists, isdir, dirname, abspath
from os.path import join, basename, exists, isdir, dirname
from time import time
class PackageImage(Image):
......
......@@ -20,8 +20,80 @@
InstallSystems repository package
'''
from installsystems.repository.manager import RepositoryManager
from installsystems.repository.config import RepositoryConfig
from installsystems.repository.repository import Repository
from installsystems.repository.repository1 import Repository1
from installsystems.repository.repository2 import Repository2
__all__ = [
"is_name",
"check_name",
"split_path",
"split_list",
"diff",
]
from installsystems.exception import ISError
from installsystems.printer import arrow, out
from re import match, split
def is_name(name):
'''Check if name is a valid repository name'''
return match("^[-_\w]+$", name) is not None
def check_name(name):
'''
Raise exception is repository name is invalid
'''
if not is_name(name):
raise ISError(u"Invalid repository name %s" % name)
return name
def split_path(path):
'''
Split an image path (repo/image:version)
in a tuple (repo, image, version)
'''
x = match(u"^(?:([^/:]+)/)?([^/:]+)?(?::v?([^/:]+)?)?$", path)
if x is None:
raise ISError(u"invalid image path: %s" % path)
return x.group(1, 2, 3)
def split_list(repolist, filter=None):
'''
Return a list of repository from a comma/spaces separated names of repo
'''
if filter is None:
filter = is_name
return [r for r in split("[ ,\n\t\v]+", repolist) if filter(r)]
@staticmethod
def diff(repo1, repo2):
'''
Compute a diff between two repositories
'''
arrow(u"Diff between repositories #y#%s#R# and #g#%s#R#" % (repo1.config.name,
repo2.config.name))
# Get info from databases
i_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
"SELECT md5, name, version FROM image").fetchall())
i_set1 = set(i_dict1.keys())
i_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
"SELECT md5, name, version FROM image").fetchall())
i_set2 = set(i_dict2.keys())
p_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
"SELECT md5, name FROM payload").fetchall())
p_set1 = set(p_dict1.keys())
p_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
"SELECT md5, name FROM payload").fetchall())
p_set2 = set(p_dict2.keys())
# computing diff
i_only1 = i_set1 - i_set2
i_only2 = i_set2 - i_set1
p_only1 = p_set1 - p_set2
p_only2 = p_set2 - p_set1
# printing functions
pimg = lambda r,c,m,d,: out("#%s#Image only in repository %s: %s v%s (%s)#R#" %
(c, r.config.name, d[m][0], d[m][1], m))
ppay = lambda r,c,m,d,: out("#%s#Payload only in repository %s: %s (%s)#R#" %
(c, r.config.name, d[m][0], m))
# printing image diff
for md5 in i_only1: pimg(repo1, "y", md5, i_dict1)
for md5 in p_only1: ppay(repo1, "y", md5, p_dict1)
for md5 in i_only2: pimg(repo2, "g", md5, i_dict2)
for md5 in p_only2: ppay(repo2, "g", md5, p_dict2)
......@@ -22,8 +22,8 @@ Repository configuration module
from grp import getgrnam
from installsystems.printer import warn, debug
from installsystems.repository.repository import Repository
from installsystems.tools import isfile, chrights, mkdir, compare_versions
from installsystems.repository import check_name
from installsystems.tools import islocal, chrights, mkdir, compare_versions
from os import getuid, getgid, umask, linesep
from os.path import join, abspath
from pwd import getpwnam
......@@ -37,7 +37,7 @@ class RepositoryConfig(object):
# set default value for arguments
self._valid_param = ("name", "path", "dbpath", "lastpath",
"uid", "gid", "fmod", "dmod", "offline")
self.name = Repository.check_name(name)
self.name = check_name(name)
self.path = ""
self._offline = False
self._dbpath = None
......@@ -111,7 +111,7 @@ class RepositoryConfig(object):
Set db path
'''
# dbpath must be local, sqlite3 requirement
if not isfile(value):
if not islocal(value):
raise ValueError("Database path must be local")
self._dbpath = abspath(value)
......
# -*- python -*-
# -*- coding: utf-8 -*-
# This file is part of Installsystems.
#
# Installsystems is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Installsystems is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
Database stuff
'''
import math
import os
import sqlite3
import uuid
import installsystems.tools as istools
from installsystems.exception import *
from installsystems.printer import *
class Database(object):
'''
Abstract repo database stuff
It needs to be local cause of sqlite3 which need to open a file
'''
version = 2.0
@classmethod
def create(cls, path):
arrow("Creating repository database")
# check locality
if not istools.isfile(path):
raise ISError("Database creation must be local")
path = os.path.abspath(path)
if os.path.exists(path):
raise ISError("Database already exists. Remove it before")
try:
conn = sqlite3.connect(path, isolation_level=None)
conn.execute("PRAGMA foreign_keys = ON")
conn.executescript(TEMPLATE_EMPTY_DB)
conn.execute("INSERT INTO repository values (?,?,?)",
(str(uuid.uuid4()), Database.version, "",))
conn.commit()
conn.close()
except Exception as e:
raise ISError(u"Create database failed", e)
return cls(path)
def __init__(self, path):
# check locality
if not istools.isfile(path):
raise ISError("Database must be local")
self.path = os.path.abspath(path)
if not os.path.exists(self.path):
raise ISError("Database not exists")
self.conn = sqlite3.connect(self.path, isolation_level=None)
self.conn.execute("PRAGMA foreign_keys = ON")
# get database version
try:
r = self.ask("SELECT version FROM repository").fetchone()
if r is None:
raise TypeError()
self.version = float(r[0])
except:
self.version = 1.0
if math.floor(self.version) >= math.floor(Database.version) + 1.0:
raise ISWarning(u"New database format (%s), please upgrade "
"your Installsystems version" % self.version)
# we make a query to be sure format is valid
try:
self.ask("SELECT * FROM image")
except:
debug(u"Invalid database format: %s" % self.version)
raise ISError("Invalid database format")
def begin(self):
'''
Start a db transaction
'''
self.conn.execute("BEGIN TRANSACTION")
def commit(self):
'''
Commit current db transaction
'''
self.conn.execute("COMMIT TRANSACTION")
def ask(self, sql, args=()):
'''
Ask question to db
'''
return self.conn.execute(sql, args)
TEMPLATE_EMPTY_DB = u"""
CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
version TEXT NOT NULL,
date INTEGER NOT NULL,
author TEXT,
description TEXT,
size INTEGER NOT NULL,
is_min_version INTEGER NOT NULL,
format INTEGER NOT NULL,
UNIQUE(name, version));
CREATE TABLE payload (md5 TEXT NOT NULL,
image_md5 TEXT NOT NULL REFERENCES image(md5),
name TEXT NOT NULL,
isdir INTEGER NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY(md5, image_md5));
CREATE TABLE repository (uuid TEXT NOT NULL PRIMARY KEY,
version FLOAT NOT NULL,
motd TEXT NOT NULL);
"""
......@@ -20,9 +20,7 @@
Repository Factory
'''
from installsystems.printer import debug, warn
from installsystems.exception import ISWarning, ISError
from installsystems.repository.database import Database
from installsystems.exception import ISError
from installsystems.repository.repository1 import Repository1
from installsystems.repository.repository2 import Repository2
......@@ -31,28 +29,21 @@ class RepositoryFactory(object):
Repository factory
'''
def __init__(self):
self.repo_class = {
1: Repository1,
2: Repository2,
}
def create(self, config):
db = None
if not config.offline:
try:
db = Database(config.dbpath)
except ISWarning as e:
warn('[%s]: %s' % (config.name, e))
config.offline = True
except ISError:
debug(u"Unable to load database %s" % config.dbpath)
config.offline = True
if config.offline:
debug(u"Repository %s is offline" % config.name)
if db is None:
def __new__(cls, config):
'''
Factory design pattern.
Return the right object version based on a version detector function
'''
version = cls.version(config.dbpath)
if version == 1:
return Repostory1(config)
elif version == 2:
return Repository2(config)
else:
return self.repo_class[int(db.version)](config, db)
raise ISError(u"Unsupported repository version")
@staticmethod
def version(path):
'''
Return the version of a database
'''
return 2
......@@ -22,9 +22,9 @@ Repository management module
from installsystems.exception import ISError, ISWarning
from installsystems.printer import out, debug, arrow
from installsystems.repository import split_path
from installsystems.repository.factory import RepositoryFactory
from installsystems.repository.repository import Repository
from installsystems.tools import isfile, chrights, PipeFile, compare_versions
from installsystems.tools import islocal, chrights, PipeFile, compare_versions
from installsystems.tools import time_rfc2822, human_size, strcspn
from json import dumps
from os import mkdir, access, W_OK, X_OK, unlink, stat, linesep, close
......@@ -49,13 +49,12 @@ class RepositoryManager(object):
self.filter = [] if filter is None else filter
self.search = [] if search is None else search
self.timeout = timeout or 3
self.factory = RepositoryFactory()
debug(u"Repository timeout setted to %ds" % self.timeout)
if cache_path is None:
self.cache_path = None
debug("No repository cache")
else:
if not isfile(cache_path):
if not islocal(cache_path):
raise NotImplementedError("Repository cache must be local")
self.cache_path = abspath(cache_path)
# must_path is a list of directory which must exists
......@@ -123,11 +122,11 @@ class RepositoryManager(object):
debug(u"Registering offline repository %s (%s)" % (config.path, config.name))
# we must force offline in cast of argument offline
config.offline = True
self.repos.append(self.factory.create(config))
self.repos.append(RepositoryFactory(config))
# if path is local, no needs to create a cache
elif isfile(config.path):
elif islocal(config.path):
debug(u"Registering direct repository %s (%s)" % (config.path, config.name))
self.repos.append(self.factory.create(config))
self.repos.append(RepositoryFactory(config))
# path is remote, we need to create a cache
else:
debug(u"Registering cached repository %s (%s)" % (config.path, config.name))
......@@ -194,7 +193,7 @@ class RepositoryManager(object):
# if something append bad during caching, we mark repo as offline
debug(u"Unable to cache repository %s: %s" % (config.name, e))
config.offline = True
return self.factory.create(config)
return RepositoryFactory(config)
@property
def names(self):
......@@ -241,7 +240,7 @@ class RepositoryManager(object):
raise ISError(u"No online repository")
ans = {}
for pattern in patterns:
path, image, version = Repository.split_path(pattern)
path, image, version = split_path(pattern)
if image is None:
if path is None or version is None:
image = "*"
......@@ -461,3 +460,103 @@ class RepositoryManager(object):
l.append(ln)
s = linesep.join(l)
out(s)
class Database(object):
'''
Abstract repo database stuff
It needs to be local cause of sqlite3 which need to open a file
'''
version = 2.0
@classmethod
def create(cls, path):
arrow("Creating repository database")
# check locality
if not istools.isfile(path):
raise ISError("Database creation must be local")
path = os.path.abspath(path)
if os.path.exists(path):
raise ISError("Database already exists. Remove it before")
try:
conn = sqlite3.connect(path, isolation_level=None)
conn.execute("PRAGMA foreign_keys = ON")
conn.executescript(TEMPLATE_EMPTY_DB)
conn.execute("INSERT INTO repository values (?,?,?)",
(str(uuid.uuid4()), Database.version, "",))
conn.commit()
conn.close()
except Exception as e:
raise ISError(u"Create database failed", e)
return cls(path)
def __init__(self, path):
# check locality
if not istools.isfile(path):
raise ISError("Database must be local")
self.path = os.path.abspath(path)
if not os.path.exists(self.path):
raise ISError("Database not exists")
self.conn = sqlite3.connect(self.path, isolation_level=None)
self.conn.execute("PRAGMA foreign_keys = ON")
# get database version
try:
r = self.ask("SELECT version FROM repository").fetchone()
if r is None:
raise TypeError()
self.version = float(r[0])
except:
self.version = 1.0
if math.floor(self.version) >= math.floor(Database.version) + 1.0:
raise ISWarning(u"New database format (%s), please upgrade "
"your Installsystems version" % self.version)
# we make a query to be sure format is valid
try:
self.ask("SELECT * FROM image")
except:
debug(u"Invalid database format: %s" % self.version)
raise ISError("Invalid database format")
def begin(self):
'''
Start a db transaction
'''
self.conn.execute("BEGIN TRANSACTION")
def commit(self):
'''
Commit current db transaction
'''
self.conn.execute("COMMIT TRANSACTION")
def ask(self, sql, args=()):
'''
Ask question to db
'''
return self.conn.execute(sql, args)
TEMPLATE_EMPTY_DB = u"""
CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
version TEXT NOT NULL,
date INTEGER NOT NULL,
author TEXT,
description TEXT,
size INTEGER NOT NULL,
is_min_version INTEGER NOT NULL,
format INTEGER NOT NULL,
UNIQUE(name, version));
CREATE TABLE payload (md5 TEXT NOT NULL,
image_md5 TEXT NOT NULL REFERENCES image(md5),
name TEXT NOT NULL,
isdir INTEGER NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY(md5, image_md5));
CREATE TABLE repository (uuid TEXT NOT NULL PRIMARY KEY,
version FLOAT NOT NULL,
motd TEXT NOT NULL);
"""
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment