Skip to content
Commits on Source (9)
Expand all Hide whitespace changes
Inline Side-by-side
bin/is
View file @ 5a136f19
......@@ -28,17 +28,16 @@ from installsystems.exception import ISError, ISException
from installsystems.image import PackageImage, SourceImage
from installsystems.printer import arrow, arrowlevel, setmode
from installsystems.printer import out, warn, error, debug, confirm
from installsystems.repository import Repository, RepositoryManager, RepositoryConfig
from installsystems.repository import split_list as split_repo_list, diff as repodiff
from installsystems.repository.manager import RepositoryManager
from installsystems.repository.config import RepositoryConfig
from installsystems.tools import chroot, prepare_chroot, unprepare_chroot
from installsystems.tools import isfile, smd5sum, argv
from installsystems.tools import islocal, smd5sum, argv
from os import getpid, getcwdu, chdir
from psutil import IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE
from psutil import Process, IOPRIO_CLASS_NONE
from socket import setdefaulttimeout
# used by os.path.isfile
import os
################################################################################
# Common functions
################################################################################
......@@ -51,8 +50,8 @@ def load_repositories(args):
if args.no_cache:
args.cache = None
# split filter and search in list
args.repo_filter = Repository.split_list(args.repo_filter)
args.repo_search = Repository.split_list(args.repo_search)
args.repo_filter = split_repo_list(args.repo_filter)
args.repo_search = split_repo_list(args.repo_search)
# init repo cache object
repoman = RepositoryManager(args.cache, timeout=args.repo_timeout or args.timeout,
filter=args.repo_filter, search=args.repo_search)
......@@ -79,7 +78,7 @@ def get_images(patterns, repoman, local=True, min=None, max=None):
ans = []
for pattern in patterns:
# check if image is a local file
if local and isfile(pattern) and os.path.isfile(pattern):
if local and islocal(pattern, True):
ans.append((pattern, None))
else: # we need to find image in a repository
ans += sorted(repoman.select_images([pattern]).items())
......@@ -255,7 +254,7 @@ def c_diff(args):
repoman = load_repositories(args)
if args.object[0] in repoman.onlines and args.object[1] in repoman.onlines:
try:
Repository.diff(repoman[args.object[0]], repoman[args.object[1]])
diff(repoman[args.object[0]], repoman[args.object[1]])
except IndexError as e:
raise ISError(e)
else:
......
installsystems/__init__.py.in
View file @ 5a136f19
......@@ -26,18 +26,20 @@ def git_version():
'''
from os import getcwd, chdir, devnull
from os.path import dirname
from subprocess import check_output, CalledProcessError
from subprocess import Popen, PIPE
from sys import argv
version = ""
cwd = getcwd()
try:
chdir(dirname(argv[0]))
version = check_output(["git", "describe", "--tags", "--always" ],
stdin=open(devnull, 'rb'),
stderr=open(devnull, "wb")).strip()
process = Popen(["git", "describe", "--tags", "--always" ],
stdout=PIPE,
stdin=open(devnull, 'rb'),
stderr=open(devnull, "wb"))
version = process.communicate()[0].strip()
if len(version) > 0:
version = "-" + version
except (OSError, CalledProcessError):
except OSError:
pass
finally:
chdir(cwd)
......
installsystems/image/image.py
View file @ 5a136f19
......@@ -145,7 +145,9 @@ class Image(object):
try:
# replace system modules by image loaded
# we must use the same directory and not copy it (probably C reference)
sysmodules.clear()
for module in self.modules:
if module in sysmodules:
del sysmodules[str(module)]
# sys must be in sys.module to allow loading of modules
sysmodules["sys"] = sys
sysmodules.update(self.modules)
......@@ -169,7 +171,7 @@ class Image(object):
'''
Check if @buf is a valid image name
'''
if match("^[-_\w]+$", buf) is None:
if match("^[-_.\w]+$", buf) is None:
raise ISError(u"Invalid image name %s" % buf)
# return the image name, because this function is used by ConfigObj
# validate to ensure the image name is correct
......
installsystems/image/package.py
View file @ 5a136f19
......@@ -34,7 +34,7 @@ from installsystems.tools import mkdir, abspath, time_rfc2822, human_size, argv,
from json import loads, dumps
from math import floor
from os import listdir
from os.path import join, basename, exists, isdir, dirname, abspath
from os.path import join, basename, exists, isdir, dirname
from time import time
class PackageImage(Image):
......
installsystems/image/source.py
View file @ 5a136f19
......@@ -30,7 +30,7 @@ from installsystems.image.image import Image
from installsystems.image.payload import Payload
from installsystems.image.tarball import Tarball, REGTYPE
from installsystems.printer import arrow, arrowlevel, warn, error
from installsystems.tools import PipeFile, isfile, get_compressor_path, chrights
from installsystems.tools import PipeFile, islocal, get_compressor_path, chrights
from json import dumps
from locale import getpreferredencoding
from os import stat, listdir, mkdir, umask, access, unlink, symlink, R_OK, X_OK
......@@ -63,7 +63,7 @@ class SourceImage(Image):
Create an empty source image
'''
# check local repository
if not isfile(path):
if not islocal(path):
raise NotImplementedError("SourceImage must be local")
# main path
build_path = join(path, "build")
......@@ -132,7 +132,7 @@ class SourceImage(Image):
'''
Image.__init__(self)
# check local repository
if not isfile(path):
if not islocal(path):
raise NotImplementedError("SourceImage must be local")
self.base_path = abspath(path)
for pathtype in ("build", "parser", "setup", "payload", "lib"):
......@@ -534,13 +534,14 @@ class SourceImage(Image):
# If everything is fine, the validation return True
# Else, it returns a list of (section, optname, error)
if res is not True:
for section, optname, error in flatten_errors(cp, res):
# If error is False, this mean no value as been supplied,
# so we use the default value
# Else, the check has failed
if error:
error('Wrong description file, %s %s: %s' % (section, optname, error))
for n in ("name","version", "description", "author", "is_min_version"):
for section, optname, err in flatten_errors(cp, res):
# If error, the check has failed
if err:
error(u"Wrong description file, %s %s: %s" % (section, optname, err))
# Else, no value has been supplied and there is no default value
else:
error(u"No option '%s' in section '%s'" % (optname, section[0]))
for n in ("name", "version", "description", "author", "is_min_version"):
d[n] = cp["image"][n]
d["compressor"] = {}
# set payload compressor
......@@ -609,7 +610,7 @@ BUILD_TPL = u"""# -*- python -*-
# rebuild list is empty by default
#rebuild += ["rootfs"]
# vim:set ts=4 sw=4 et:
# vim:set ts=4 sw=4 ai et:
"""
PARSER_TPL = u"""# -*- python -*-
......@@ -620,13 +621,13 @@ PARSER_TPL = u"""# -*- python -*-
# you can use exit() to break the execution of the script
import os
import argparse
from argparse import Action
from installsystems.printer import arrow
from os.path import isdir
class TargetAction(argparse.Action):
class TargetAction(Action):
def __call__(self, parser, namespace, values, option_string=None):
if not os.path.isdir(values):
if not isdir(values):
raise Exception(u"Invalid target directory %s" % values)
namespace.target = values
......@@ -634,7 +635,7 @@ parser.add_argument("-n", "--hostname", dest="hostname", type=str, required=True
parser.add_argument("target", type=str, action=TargetAction,
help="target installation directory")
# vim:set ts=4 sw=4 et:
# vim:set ts=4 sw=4 ai et:
"""
SETUP_TPL = u"""# -*- python -*-
......@@ -652,7 +653,7 @@ arrow(u"hostname: %s" % namespace.hostname)
# uncomment to extract payload named root in namespace.target directory
#image.payload["rootfs"].extract(namespace.target)
# vim:set ts=4 sw=4 et:
# vim:set ts=4 sw=4 ai et:
"""
# This must not be an unicode string, because configobj don't decode configspec
......@@ -661,9 +662,9 @@ DESCRIPTION_CONFIG_SPEC = """\
[image]
name = IS_name
version = IS_version
description = string
author = string
is_min_version = IS_min_version
description = string(default='')
author = string(default='')
is_min_version = IS_min_version(default=0)
[compressor]
__many__ = force_list
......
installsystems/repository/__init__.py
View file @ 5a136f19
......@@ -20,8 +20,80 @@
InstallSystems repository package
'''
from installsystems.repository.manager import RepositoryManager
from installsystems.repository.config import RepositoryConfig
from installsystems.repository.repository import Repository
from installsystems.repository.repository1 import Repository1
from installsystems.repository.repository2 import Repository2
__all__ = [
"is_name",
"check_name",
"split_path",
"split_list",
"diff",
]
from installsystems.exception import ISError
from installsystems.printer import arrow, out
from re import match, split
def is_name(name):
'''Check if name is a valid repository name'''
return match("^[-_\w]+$", name) is not None
def check_name(name):
'''
Raise exception is repository name is invalid
'''
if not is_name(name):
raise ISError(u"Invalid repository name %s" % name)
return name
def split_path(path):
'''
Split an image path (repo/image:version)
in a tuple (repo, image, version)
'''
x = match(u"^(?:([^/:]+)/)?([^/:]+)?(?::v?([^/:]+)?)?$", path)
if x is None:
raise ISError(u"invalid image path: %s" % path)
return x.group(1, 2, 3)
def split_list(repolist, filter=None):
'''
Return a list of repository from a comma/spaces separated names of repo
'''
if filter is None:
filter = is_name
return [r for r in split("[ ,\n\t\v]+", repolist) if filter(r)]
@staticmethod
def diff(repo1, repo2):
'''
Compute a diff between two repositories
'''
arrow(u"Diff between repositories #y#%s#R# and #g#%s#R#" % (repo1.config.name,
repo2.config.name))
# Get info from databases
i_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
"SELECT md5, name, version FROM image").fetchall())
i_set1 = set(i_dict1.keys())
i_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
"SELECT md5, name, version FROM image").fetchall())
i_set2 = set(i_dict2.keys())
p_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
"SELECT md5, name FROM payload").fetchall())
p_set1 = set(p_dict1.keys())
p_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
"SELECT md5, name FROM payload").fetchall())
p_set2 = set(p_dict2.keys())
# computing diff
i_only1 = i_set1 - i_set2
i_only2 = i_set2 - i_set1
p_only1 = p_set1 - p_set2
p_only2 = p_set2 - p_set1
# printing functions
pimg = lambda r,c,m,d,: out("#%s#Image only in repository %s: %s v%s (%s)#R#" %
(c, r.config.name, d[m][0], d[m][1], m))
ppay = lambda r,c,m,d,: out("#%s#Payload only in repository %s: %s (%s)#R#" %
(c, r.config.name, d[m][0], m))
# printing image diff
for md5 in i_only1: pimg(repo1, "y", md5, i_dict1)
for md5 in p_only1: ppay(repo1, "y", md5, p_dict1)
for md5 in i_only2: pimg(repo2, "g", md5, i_dict2)
for md5 in p_only2: ppay(repo2, "g", md5, p_dict2)
installsystems/repository/config.py
View file @ 5a136f19
......@@ -22,8 +22,8 @@ Repository configuration module
from grp import getgrnam
from installsystems.printer import warn, debug
from installsystems.repository.repository import Repository
from installsystems.tools import isfile, chrights, mkdir, compare_versions
from installsystems.repository import check_name
from installsystems.tools import islocal, chrights, mkdir, compare_versions
from os import getuid, getgid, umask, linesep
from os.path import join, abspath
from pwd import getpwnam
......@@ -37,7 +37,7 @@ class RepositoryConfig(object):
# set default value for arguments
self._valid_param = ("name", "path", "dbpath", "lastpath",
"uid", "gid", "fmod", "dmod", "offline")
self.name = Repository.check_name(name)
self.name = check_name(name)
self.path = ""
self._offline = False
self._dbpath = None
......@@ -111,7 +111,7 @@ class RepositoryConfig(object):
Set db path
'''
# dbpath must be local, sqlite3 requirement
if not isfile(value):
if not islocal(value):
raise ValueError("Database path must be local")
self._dbpath = abspath(value)
......
installsystems/repository/database.py deleted 100644 → 0
View file @ 23a3b22e
# -*- python -*-
# -*- coding: utf-8 -*-
# This file is part of Installsystems.
#
# Installsystems is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Installsystems is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
Database stuff
'''
import math
import os
import sqlite3
import uuid
import installsystems.tools as istools
from installsystems.exception import *
from installsystems.printer import *
class Database(object):
'''
Abstract repo database stuff
It needs to be local cause of sqlite3 which need to open a file
'''
version = 2.0
@classmethod
def create(cls, path):
arrow("Creating repository database")
# check locality
if not istools.isfile(path):
raise ISError("Database creation must be local")
path = os.path.abspath(path)
if os.path.exists(path):
raise ISError("Database already exists. Remove it before")
try:
conn = sqlite3.connect(path, isolation_level=None)
conn.execute("PRAGMA foreign_keys = ON")
conn.executescript(TEMPLATE_EMPTY_DB)
conn.execute("INSERT INTO repository values (?,?,?)",
(str(uuid.uuid4()), Database.version, "",))
conn.commit()
conn.close()
except Exception as e:
raise ISError(u"Create database failed", e)
return cls(path)
def __init__(self, path):
# check locality
if not istools.isfile(path):
raise ISError("Database must be local")
self.path = os.path.abspath(path)
if not os.path.exists(self.path):
raise ISError("Database not exists")
self.conn = sqlite3.connect(self.path, isolation_level=None)
self.conn.execute("PRAGMA foreign_keys = ON")
# get database version
try:
r = self.ask("SELECT version FROM repository").fetchone()
if r is None:
raise TypeError()
self.version = float(r[0])
except:
self.version = 1.0
if math.floor(self.version) >= math.floor(Database.version) + 1.0:
raise ISWarning(u"New database format (%s), please upgrade "
"your Installsystems version" % self.version)
# we make a query to be sure format is valid
try:
self.ask("SELECT * FROM image")
except:
debug(u"Invalid database format: %s" % self.version)
raise ISError("Invalid database format")
def begin(self):
'''
Start a db transaction
'''
self.conn.execute("BEGIN TRANSACTION")
def commit(self):
'''
Commit current db transaction
'''
self.conn.execute("COMMIT TRANSACTION")
def ask(self, sql, args=()):
'''
Ask question to db
'''
return self.conn.execute(sql, args)
TEMPLATE_EMPTY_DB = u"""
CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
version TEXT NOT NULL,
date INTEGER NOT NULL,
author TEXT,
description TEXT,
size INTEGER NOT NULL,
is_min_version INTEGER NOT NULL,
format INTEGER NOT NULL,
UNIQUE(name, version));
CREATE TABLE payload (md5 TEXT NOT NULL,
image_md5 TEXT NOT NULL REFERENCES image(md5),
name TEXT NOT NULL,
isdir INTEGER NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY(md5, image_md5));
CREATE TABLE repository (uuid TEXT NOT NULL PRIMARY KEY,
version FLOAT NOT NULL,
motd TEXT NOT NULL);
"""
installsystems/repository/factory.py
View file @ 5a136f19
......@@ -20,9 +20,7 @@
Repository Factory
'''
from installsystems.printer import debug, warn
from installsystems.exception import ISWarning, ISError
from installsystems.repository.database import Database
from installsystems.exception import ISError
from installsystems.repository.repository1 import Repository1
from installsystems.repository.repository2 import Repository2
......@@ -31,28 +29,21 @@ class RepositoryFactory(object):
Repository factory
'''
def __init__(self):
self.repo_class = {
1: Repository1,
2: Repository2,
}
def create(self, config):
db = None
if not config.offline:
try:
db = Database(config.dbpath)
except ISWarning as e:
warn('[%s]: %s' % (config.name, e))
config.offline = True
except ISError:
debug(u"Unable to load database %s" % config.dbpath)
config.offline = True
if config.offline:
debug(u"Repository %s is offline" % config.name)
if db is None:
def __new__(cls, config):
'''
Factory design pattern.
Return the right object version based on a version detector function
'''
version = cls.version(config.dbpath)
if version == 1:
return Repostory1(config)
elif version == 2:
return Repository2(config)
else:
return self.repo_class[int(db.version)](config, db)
raise ISError(u"Unsupported repository version")
@staticmethod
def version(path):
'''
Return the version of a database
'''
return 2
installsystems/repository/manager.py
View file @ 5a136f19
......@@ -22,9 +22,9 @@ Repository management module
from installsystems.exception import ISError, ISWarning
from installsystems.printer import out, debug, arrow
from installsystems.repository import split_path
from installsystems.repository.factory import RepositoryFactory
from installsystems.repository.repository import Repository
from installsystems.tools import isfile, chrights, PipeFile, compare_versions
from installsystems.tools import islocal, chrights, PipeFile, compare_versions
from installsystems.tools import time_rfc2822, human_size, strcspn
from json import dumps
from os import mkdir, access, W_OK, X_OK, unlink, stat, linesep, close
......@@ -49,13 +49,12 @@ class RepositoryManager(object):
self.filter = [] if filter is None else filter
self.search = [] if search is None else search
self.timeout = timeout or 3
self.factory = RepositoryFactory()
debug(u"Repository timeout setted to %ds" % self.timeout)
if cache_path is None:
self.cache_path = None
debug("No repository cache")
else:
if not isfile(cache_path):
if not islocal(cache_path):
raise NotImplementedError("Repository cache must be local")
self.cache_path = abspath(cache_path)
# must_path is a list of directory which must exists
......@@ -123,11 +122,11 @@ class RepositoryManager(object):
debug(u"Registering offline repository %s (%s)" % (config.path, config.name))
# we must force offline in cast of argument offline
config.offline = True
self.repos.append(self.factory.create(config))
self.repos.append(RepositoryFactory(config))
# if path is local, no needs to create a cache
elif isfile(config.path):
elif islocal(config.path):
debug(u"Registering direct repository %s (%s)" % (config.path, config.name))
self.repos.append(self.factory.create(config))
self.repos.append(RepositoryFactory(config))
# path is remote, we need to create a cache
else:
debug(u"Registering cached repository %s (%s)" % (config.path, config.name))
......@@ -194,7 +193,7 @@ class RepositoryManager(object):
# if something append bad during caching, we mark repo as offline
debug(u"Unable to cache repository %s: %s" % (config.name, e))
config.offline = True
return self.factory.create(config)
return RepositoryFactory(config)
@property
def names(self):
......@@ -241,7 +240,7 @@ class RepositoryManager(object):
raise ISError(u"No online repository")
ans = {}
for pattern in patterns:
path, image, version = Repository.split_path(pattern)
path, image, version = split_path(pattern)
if image is None:
if path is None or version is None:
image = "*"
......@@ -461,3 +460,103 @@ class RepositoryManager(object):
l.append(ln)
s = linesep.join(l)
out(s)
class Database(object):
'''
Abstract repo database stuff
It needs to be local cause of sqlite3 which need to open a file
'''
version = 2.0
@classmethod
def create(cls, path):
arrow("Creating repository database")
# check locality
if not istools.isfile(path):
raise ISError("Database creation must be local")
path = os.path.abspath(path)
if os.path.exists(path):
raise ISError("Database already exists. Remove it before")
try:
conn = sqlite3.connect(path, isolation_level=None)
conn.execute("PRAGMA foreign_keys = ON")
conn.executescript(TEMPLATE_EMPTY_DB)
conn.execute("INSERT INTO repository values (?,?,?)",
(str(uuid.uuid4()), Database.version, "",))
conn.commit()
conn.close()
except Exception as e:
raise ISError(u"Create database failed", e)
return cls(path)
def __init__(self, path):
# check locality
if not istools.isfile(path):
raise ISError("Database must be local")
self.path = os.path.abspath(path)
if not os.path.exists(self.path):
raise ISError("Database not exists")
self.conn = sqlite3.connect(self.path, isolation_level=None)
self.conn.execute("PRAGMA foreign_keys = ON")
# get database version
try:
r = self.ask("SELECT version FROM repository").fetchone()
if r is None:
raise TypeError()
self.version = float(r[0])
except:
self.version = 1.0
if math.floor(self.version) >= math.floor(Database.version) + 1.0:
raise ISWarning(u"New database format (%s), please upgrade "
"your Installsystems version" % self.version)
# we make a query to be sure format is valid
try:
self.ask("SELECT * FROM image")
except:
debug(u"Invalid database format: %s" % self.version)
raise ISError("Invalid database format")
def begin(self):
'''
Start a db transaction
'''
self.conn.execute("BEGIN TRANSACTION")
def commit(self):
'''
Commit current db transaction
'''
self.conn.execute("COMMIT TRANSACTION")
def ask(self, sql, args=()):
'''
Ask question to db
'''
return self.conn.execute(sql, args)
TEMPLATE_EMPTY_DB = u"""
CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
version TEXT NOT NULL,
date INTEGER NOT NULL,
author TEXT,
description TEXT,
size INTEGER NOT NULL,
is_min_version INTEGER NOT NULL,
format INTEGER NOT NULL,
UNIQUE(name, version));
CREATE TABLE payload (md5 TEXT NOT NULL,
image_md5 TEXT NOT NULL REFERENCES image(md5),
name TEXT NOT NULL,
isdir INTEGER NOT NULL,
size INTEGER NOT NULL,
PRIMARY KEY(md5, image_md5));
CREATE TABLE repository (uuid TEXT NOT NULL PRIMARY KEY,
version FLOAT NOT NULL,
motd TEXT NOT NULL);
"""
installsystems/repository/repository.py deleted 100644 → 0
View file @ 23a3b22e
# -*- python -*-
# -*- coding: utf-8 -*-
# This file is part of Installsystems.
#
# Installsystems is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Installsystems is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Installsystems. If not, see <http://www.gnu.org/licenses/>.
'''
Abstract repository module
'''
from cStringIO import StringIO
from installsystems.exception import ISError
from installsystems.image.package import PackageImage
from installsystems.printer import arrow, arrowlevel, out, warn, confirm
from installsystems.repository.database import Database
from installsystems.tools import isfile, chrights, mkdir, compare_versions, PipeFile
from os import unlink, listdir, linesep, rmdir
from os.path import join
from os.path import join, basename, exists, isdir
from re import match, split
from time import time
class Repository(object):
'''
Repository class
'''
@staticmethod
def is_name(name):
'''Check if name is a valid repository name'''
return match("^[-_\w]+$", name) is not None
@staticmethod
def check_name(name):
'''
Raise exception is repository name is invalid
'''
if not Repository.is_name(name):
raise ISError(u"Invalid repository name %s" % name)
return name
@staticmethod
def split_path(path):
'''
Split an image path (repo/image:version)
in a tuple (repo, image, version)
'''
x = match(u"^(?:([^/:]+)/)?([^/:]+)?(?::v?([^/:]+)?)?$", path)
if x is None:
raise ISError(u"invalid image path: %s" % path)
return x.group(1, 2, 3)
@staticmethod
def split_list(repolist, filter=None):
'''
Return a list of repository from a comma/spaces separated names of repo
'''
if filter is None:
filter = Repository.is_name
return [r for r in split("[ ,\n\t\v]+", repolist) if filter(r)]
@staticmethod
def diff(repo1, repo2):
'''
Compute a diff between two repositories
'''
arrow(u"Diff between repositories #y#%s#R# and #g#%s#R#" % (repo1.config.name,
repo2.config.name))
# Get info from databases
i_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
"SELECT md5, name, version FROM image").fetchall())
i_set1 = set(i_dict1.keys())
i_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
"SELECT md5, name, version FROM image").fetchall())
i_set2 = set(i_dict2.keys())
p_dict1 = dict((b[0], b[1:]) for b in repo1.db.ask(
"SELECT md5, name FROM payload").fetchall())
p_set1 = set(p_dict1.keys())
p_dict2 = dict((b[0], b[1:]) for b in repo2.db.ask(
"SELECT md5, name FROM payload").fetchall())
p_set2 = set(p_dict2.keys())
# computing diff
i_only1 = i_set1 - i_set2
i_only2 = i_set2 - i_set1
p_only1 = p_set1 - p_set2
p_only2 = p_set2 - p_set1
# printing functions
pimg = lambda r,c,m,d,: out("#%s#Image only in repository %s: %s v%s (%s)#R#" %
(c, r.config.name, d[m][0], d[m][1], m))
ppay = lambda r,c,m,d,: out("#%s#Payload only in repository %s: %s (%s)#R#" %
(c, r.config.name, d[m][0], m))
# printing image diff
for md5 in i_only1: pimg(repo1, "y", md5, i_dict1)
for md5 in p_only1: ppay(repo1, "y", md5, p_dict1)
for md5 in i_only2: pimg(repo2, "g", md5, i_dict2)
for md5 in p_only2: ppay(repo2, "g", md5, p_dict2)
def __init__(self, config, db=None):
self.config = config
self.local = isfile(self.config.path)
self.db = db
def __getattribute__(self, name):
'''
Raise an error if repository is unavailable
Unavailable can be caused because db is not accessible or
because repository is not initialized
'''
config = object.__getattribute__(self, "config")
# config, init, local and upgrade are always accessible
if name in ("init", "config", "local", "upgrade"):
return object.__getattribute__(self, name)
# if no db (not init or not accessible) raise error
if config.offline:
raise ISError(u"Repository %s is offline" % config.name)
return object.__getattribute__(self, name)
@property
def version(self):
'''
Return repository version
'''
raise NotImplementedError()
@property
def uuid(self):
'''
Return repository UUID
'''
return self.db.ask("SELECT uuid from repository").fetchone()[0]
def init(self):
'''
Initialize an empty base repository
'''
config = self.config
# check local repository
if not self.local:
raise ISError(u"Repository creation must be local")
# create base directories
arrow("Creating base directories")
arrowlevel(1)
# creating local directory
try:
if exists(config.path):
arrow(u"%s already exists" % config.path)
else:
mkdir(config.path, config.uid, config.gid, config.dmod)
arrow(u"%s directory created" % config.path)
except Exception as e:
raise ISError(u"Unable to create directory %s" % config.path, e)
arrowlevel(-1)
# create database
d = Database.create(config.dbpath)
chrights(config.dbpath, uid=config.uid,
gid=config.gid, mode=config.fmod)
# load database
self.db = Database(config.dbpath)
# mark repo as not offline
self.config.offline = False
# create/update last file
self.update_last()
def update_last(self):
'''
Update last file to current time
'''
# check local repository
if not self.local:
raise ISError(u"Repository must be local")
try:
arrow("Updating last file")
last_path = join(self.config.path, self.config.lastname)
open(last_path, "w").write("%s\n" % int(time()))
chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
except Exception as e:
raise ISError(u"Update last file failed", e)
def last(self, name):
'''
Return last version of name in repo or None if not found
'''
r = self.db.ask("SELECT version FROM image WHERE name = ?", (name,)).fetchall()
# no row => no way
if r is None:
return None
f = lambda x,y: x[0] if compare_versions(x[0], y[0]) > 0 else y[0]
# return last
return reduce(f, r)
def _add(self, image):
'''
Add description to db
'''
arrow("Adding metadata")
self.db.begin()
# insert image information
arrow("Image", 1)
self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?,?,?)",
(image.md5,
image.name,
image.version,
image.date,
image.author,
image.description,
image.size,
image.is_min_version,
image.format,
))
# insert data information
arrow("Payloads", 1)
for name, obj in image.payload.items():
self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
(obj.md5,
image.md5,
name,
obj.isdir,
obj.size,
))
# on commit
self.db.commit()
# update last file
self.update_last()
def add(self, image, delete=False):
'''
Add a packaged image to repository
if delete is true, remove original files
'''
# check local repository
if not self.local:
raise ISError(u"Repository addition must be local")
# cannot add already existant image
if self.has(image.name, image.version):
raise ISError(u"Image already in database, delete first!")
# adding file to repository
arrow("Copying images and payload")
for obj in [ image ] + image.payload.values():
dest = join(self.config.path, obj.md5)
basesrc = basename(obj.path)
if exists(dest):
arrow(u"Skipping %s: already exists" % basesrc, 1)
else:
arrow(u"Adding %s (%s)" % (basesrc, obj.md5), 1)
dfo = open(dest, "wb")
sfo = PipeFile(obj.path, "r", progressbar=True)
sfo.consume(dfo)
sfo.close()
dfo.close()
chrights(dest, self.config.uid,
self.config.gid, self.config.fmod)
# copy is done. create a image inside repo
r_image = PackageImage(join(self.config.path, image.md5),
md5name=True)
# checking must be done with original md5
r_image.md5 = image.md5
# checking image and payload after copy
r_image.check("Check image and payload")
self._add(image)
# removing orginal files
if delete:
arrow("Removing original files")
for obj in [ image ] + image.payload.values():
arrow(basename(obj.path), 1)
unlink(obj.path)
def getallmd5(self):
'''
Get list of all md5 in DB
'''
res = self.db.ask("SELECT md5 FROM image UNION SELECT md5 FROM payload").fetchall()
return [ md5[0] for md5 in res ]
def check(self):
'''
Check repository for unreferenced and missing files
'''
# Check if the repo is local
if not self.local:
raise ISError(u"Repository must be local")
local_files = set(listdir(self.config.path))
local_files.remove(self.config.dbname)
local_files.remove(self.config.lastname)
db_files = set(self.getallmd5())
# check missing files
arrow("Checking missing files")
missing_files = db_files - local_files
if len(missing_files) > 0:
out(linesep.join(missing_files))
# check unreferenced files
arrow("Checking unreferenced files")
unref_files = local_files - db_files
if len(unref_files) > 0:
out(linesep.join(unref_files))
# check corruption of local files
arrow("Checking corrupted files")
for f in local_files:
fo = PipeFile(join(self.config.path, f))
fo.consume()
fo.close()
if fo.md5 != f:
out(f)
def clean(self, force=False):
'''
Clean the repository's content
'''
# Check if the repo is local
if not self.local:
raise ISError(u"Repository must be local")
allmd5 = set(self.getallmd5())
repofiles = set(listdir(self.config.path)) - set([self.config.dbname, self.config.lastname])
dirtyfiles = repofiles - allmd5
if len(dirtyfiles) > 0:
# print dirty files
arrow("Dirty files:")
for f in dirtyfiles:
arrow(f, 1)
# ask confirmation
if not force and not confirm("Remove dirty files? (yes) "):
raise ISError(u"Aborted!")
# start cleaning
arrow("Cleaning")
for f in dirtyfiles:
p = join(self.config.path, f)
arrow(u"Removing %s" % p, 1)
try:
if isdir(p):
rmdir(p)
else:
unlink(p)
except:
warn(u"Removing %s failed" % p)
else:
arrow("Nothing to clean")
def delete(self, name, version, payloads=True):
'''
Delete an image from repository
'''
# check local repository
if not self.local:
raise ISError(u"Repository deletion must be local")
# get md5 of files related to images (exception is raised if not exists
md5s = self.getmd5(name, version)
# cleaning db (must be done before cleaning)
arrow("Cleaning database")
arrow("Remove payloads from database", 1)
self.db.begin()
for md5 in md5s[1:]:
self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?",
(md5, md5s[0])).fetchone()
arrow("Remove image from database", 1)
self.db.ask("DELETE FROM image WHERE md5 = ?",
(md5s[0],)).fetchone()
self.db.commit()
# Removing files
arrow("Removing files from pool")
# if asked don't remove payloads
if not payloads:
md5s = [ md5s[0] ]
arrowlevel(1)
for md5 in md5s:
self._remove_file(md5)
arrowlevel(-1)
# update last file
self.update_last()
def images(self):
'''
Return a dict of information on images
'''
db_images = self.db.ask("SELECT md5, name, version, date, author, \
description, size, is_min_version, format \
FROM image ORDER BY name, version").fetchall()
images = []
field = ("md5", "name", "version", "date", "author", "description",
"size", "is_min_version", "format")
for info in db_images:
d = dict(zip(field, info))
d["repo"] = self.config.name
d["url"] = join(self.config.path, d["md5"])
images.append(d)
return images
def payloads(self):
'''
Return a dict of information on payloads
'''
db_payloads = self.db.ask("SELECT payload.md5,payload.size,payload.isdir,image.name,image.version,payload.name FROM payload inner join image on payload.image_md5 = image.md5").fetchall()
res = {}
for payload in db_payloads:
md5 = payload[0]
# create entry if not exists
if md5 not in res:
res[md5] = {"size": payload[1], "isdir": payload[2], "images": {}}
# add image to list
imgpath = u"%s/%s:%s" % (self.config.name, payload[3], payload[4])
res[md5]["images"][imgpath] = {"repo": self.config.name,
"imgname": payload[3],
"imgver": payload[4],
"payname": payload[5]}
return res
def search(self, pattern):
'''
Search pattern in a repository
'''
images = self.db.ask("SELECT name, version, author, description\
FROM image\
WHERE name LIKE ? OR\
description LIKE ? OR\
author LIKE ?",
tuple( [u"%%%s%%" % pattern ] * 3)
).fetchall()
for name, version, author, description in images:
arrow(u"%s v%s" % (name, version), 1)
out(u" #yellow#Author:#reset# %s" % author)
out(u" #yellow#Description:#reset# %s" % description)
def _remove_file(self, filename):
'''
Remove a filename from pool. Check if it's not needed by db before
'''
# check existance in table image
have = False
for table in ("image", "payload"):
have = have or self.db.ask(u"SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table,
(filename,)).fetchone() is not None
# if no reference, delete!
if not have:
arrow(u"%s, deleted" % filename)
unlink(join(self.config.path, filename))
else:
arrow(u"%s, skipped" % filename)
def has(self, name, version):
'''
Return the existance of a package
'''
return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None
def get(self, name, version=None):
'''
Return an image from a name and version
'''
# is no version take the last
if version is None:
version = self.last(name)
if version is None:
raise ISError(u"Unable to find image %s in %s" % (name,
self.config.name))
# get file md5 from db
r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
(name, version)).fetchone()
if r is None:
raise ISError(u"Unable to find image %s v%s in %s" % (name, version,
self.config.name))
path = join(self.config.path, r[0])
# getting the file
arrow(u"Loading image %s v%s from repository %s" % (name,
version,
self.config.name))
memfile = StringIO()
try:
fo = PipeFile(path, "r")
fo.consume(memfile)
fo.close()
except Exception as e:
raise ISError(u"Loading image %s v%s failed" % (name, version), e)
memfile.seek(0)
pkg = PackageImage(path, fileobj=memfile, md5name=True)
if pkg.md5 != r[0]:
raise ISError(u"Image MD5 verification failure")
return pkg
def getmd5(self, name, version):
'''
Return an image md5 and payload md5 from name and version. Order matter !
Image md5 will still be the first
'''
# get file md5 from db
a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1",
(name,version)).fetchone()
if a is None:
raise ISError(u"No such image %s version %s" % (name, version))
b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?",
(a[0],)).fetchall()
return [ a[0] ] + [ x[0] for x in b ]
@property
def motd(self):
'''
Return repository message of the day
'''
motd = self.db.ask("SELECT motd FROM repository").fetchone()[0]
return None if len(motd) == 0 else motd
def setmotd(self, value=""):
'''
Set repository message of the day
'''
# check local repository
if not self.local:
raise ISError(u"Repository must be local")
arrow("Updating motd")
self.db.ask("UPDATE repository SET motd = ?", (value,))
self.update_last()
installsystems/repository/repository1.py
View file @ 5a136f19
This diff is collapsed.
installsystems/repository/repository2.py
View file @ 5a136f19
......@@ -26,7 +26,7 @@ from installsystems.image.package import PackageImage
from installsystems.printer import arrow, arrowlevel, warn, info, out, confirm
from installsystems.repository.database import Database
from installsystems.repository.repository1 import Repository1
from installsystems.tools import PipeFile, isfile, chrights, mkdir, compare_versions
from installsystems.tools import PipeFile, islocal, chrights, mkdir, compare_versions
from os import unlink, listdir, linesep, rmdir, symlink
from os.path import join, exists, basename, isdir
from shutil import move, rmtree
......@@ -36,6 +36,285 @@ class Repository2(Repository1):
Repository class
'''
def __init__(self, config, db=None):
self.config = config
self.local = islocal(self.config.path)
self.db = db
def __getattribute__(self, name):
'''
Raise an error if repository is unavailable
Unavailable can be caused because db is not accessible or
because repository is not initialized
'''
config = object.__getattribute__(self, "config")
# config, init, local and upgrade are always accessible
if name in ("init", "config", "local"):
return object.__getattribute__(self, name)
# if no db (not init or not accessible) raise error
if config.offline:
raise ISError(u"Repository %s is offline" % config.name)
return object.__getattribute__(self, name)
def update_last(self):
'''
Update last file to current time
'''
# check local repository
if not self.local:
raise ISError(u"Repository must be local")
try:
arrow("Updating last file")
last_path = join(self.config.path, self.config.lastname)
open(last_path, "w").write("%s\n" % int(time()))
chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
except Exception as e:
raise ISError(u"Update last file failed", e)
def last(self, name):
'''
Return last version of name in repo or None if not found
'''
r = self.db.ask("SELECT version FROM image WHERE name = ?", (name,)).fetchall()
# no row => no way
if r is None:
return None
f = lambda x,y: x[0] if compare_versions(x[0], y[0]) > 0 else y[0]
# return last
return reduce(f, r)
def add(self, image, delete=False):
'''
Add a packaged image to repository
if delete is true, remove source files
'''
raise NotImplementedError()
def getallmd5(self):
'''
Get list of all md5 in DB
'''
res = self.db.ask("SELECT md5 FROM image UNION SELECT md5 FROM payload").fetchall()
return [ md5[0] for md5 in res ]
def check(self):
'''
Check repository for unreferenced and missing files
'''
# Check if the repo is local
if not self.local:
raise ISError(u"Repository must be local")
local_files = set(listdir(self.config.path))
local_files.remove(self.config.dbname)
local_files.remove(self.config.lastname)
db_files = set(self.getallmd5())
# check missing files
arrow("Checking missing files")
missing_files = db_files - local_files
if len(missing_files) > 0:
out(linesep.join(missing_files))
# check unreferenced files
arrow("Checking unreferenced files")
unref_files = local_files - db_files
if len(unref_files) > 0:
out(linesep.join(unref_files))
# check corruption of local files
arrow("Checking corrupted files")
for f in local_files:
fo = PipeFile(join(self.config.path, f))
fo.consume()
fo.close()
if fo.md5 != f:
out(f)
def clean(self, force=False):
'''
Clean the repository's content
'''
# Check if the repo is local
if not self.local:
raise ISError(u"Repository must be local")
allmd5 = set(self.getallmd5())
repofiles = set(listdir(self.config.path)) - set([self.config.dbname, self.config.lastname])
dirtyfiles = repofiles - allmd5
if len(dirtyfiles) > 0:
# print dirty files
arrow("Dirty files:")
for f in dirtyfiles:
arrow(f, 1)
# ask confirmation
if not force and not confirm("Remove dirty files? (yes) "):
raise ISError(u"Aborted!")
# start cleaning
arrow("Cleaning")
for f in dirtyfiles:
p = join(self.config.path, f)
arrow(u"Removing %s" % p, 1)
try:
if isdir(p):
rmdir(p)
else:
unlink(p)
except:
warn(u"Removing %s failed" % p)
else:
arrow("Nothing to clean")
def delete(self, name, version, payloads=True):
'''
Delete an image from repository
'''
# check local repository
if not self.local:
raise ISError(u"Repository deletion must be local")
# get md5 of files related to images (exception is raised if not exists
md5s = self.getmd5(name, version)
# cleaning db (must be done before cleaning)
arrow("Cleaning database")
arrow("Remove payloads from database", 1)
self.db.begin()
for md5 in md5s[1:]:
self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?",
(md5, md5s[0])).fetchone()
arrow("Remove image from database", 1)
self.db.ask("DELETE FROM image WHERE md5 = ?",
(md5s[0],)).fetchone()
self.db.commit()
# Removing files
arrow("Removing files from pool")
# if asked don't remove payloads
if not payloads:
md5s = [ md5s[0] ]
arrowlevel(1)
for md5 in md5s:
self._remove_file(md5)
arrowlevel(-1)
# update last file
self.update_last()
def images(self):
'''
Return a dict of information on images
'''
db_images = self.db.ask("SELECT md5, name, version, date, author, \
description, size, is_min_version, format \
FROM image ORDER BY name, version").fetchall()
images = []
field = ("md5", "name", "version", "date", "author", "description",
"size", "is_min_version", "format")
for info in db_images:
d = dict(zip(field, info))
d["repo"] = self.config.name
d["url"] = join(self.config.path, d["md5"])
images.append(d)
return images
def payloads(self):
'''
Return a dict of information on payloads
'''
db_payloads = self.db.ask("SELECT payload.md5,payload.size,payload.isdir,image.name,image.version,payload.name FROM payload inner join image on payload.image_md5 = image.md5").fetchall()
res = {}
for payload in db_payloads:
md5 = payload[0]
# create entry if not exists
if md5 not in res:
res[md5] = {"size": payload[1], "isdir": payload[2], "images": {}}
# add image to list
imgpath = u"%s/%s:%s" % (self.config.name, payload[3], payload[4])
res[md5]["images"][imgpath] = {"repo": self.config.name,
"imgname": payload[3],
"imgver": payload[4],
"payname": payload[5]}
return res
def search(self, pattern):
'''
Search pattern in a repository
'''
images = self.db.ask("SELECT name, version, author, description\
FROM image\
WHERE name LIKE ? OR\
description LIKE ? OR\
author LIKE ?",
tuple( [u"%%%s%%" % pattern ] * 3)
).fetchall()
for name, version, author, description in images:
arrow(u"%s v%s" % (name, version), 1)
out(u" #yellow#Author:#reset# %s" % author)
out(u" #yellow#Description:#reset# %s" % description)
def _remove_file(self, filename):
'''
Remove a filename from pool. Check if it's not needed by db before
'''
# check existance in table image
have = False
for table in ("image", "payload"):
have = have or self.db.ask(u"SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table,
(filename,)).fetchone() is not None
# if no reference, delete!
if not have:
arrow(u"%s, deleted" % filename)
unlink(join(self.config.path, filename))
else:
arrow(u"%s, skipped" % filename)
def has(self, name, version):
'''
Return the existance of a package
'''
return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None
def get(self, name, version=None):
'''
Return an image from a name and version
'''
# is no version take the last
if version is None:
version = self.last(name)
if version is None:
raise ISError(u"Unable to find image %s in %s" % (name,
self.config.name))
# get file md5 from db
r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
(name, version)).fetchone()
if r is None:
raise ISError(u"Unable to find image %s v%s in %s" % (name, version,
self.config.name))
path = join(self.config.path, r[0])
# getting the file
arrow(u"Loading image %s v%s from repository %s" % (name,
version,
self.config.name))
memfile = StringIO()
try:
fo = PipeFile(path, "r")
fo.consume(memfile)
fo.close()
except Exception as e:
raise ISError(u"Loading image %s v%s failed" % (name, version), e)
memfile.seek(0)
pkg = PackageImage(path, fileobj=memfile, md5name=True)
if pkg.md5 != r[0]:
raise ISError(u"Image MD5 verification failure")
return pkg
def getmd5(self, name, version):
'''
Return an image md5 and payload md5 from name and version. Order matter !
Image md5 will still be the first
'''
# get file md5 from db
a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1",
(name,version)).fetchone()
if a is None:
raise ISError(u"No such image %s version %s" % (name, version))
b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?",
(a[0],)).fetchall()
return [ a[0] ] + [ x[0] for x in b ]
@property
def version(self):
'''
......@@ -416,4 +695,8 @@ class Repository2(Repository1):
self.update_last()
def upgrade(self):
'''
Upgrade database to the next version
'''
info("No upgrade available")
installsystems/tools.py
View file @ 5a136f19
......@@ -325,18 +325,35 @@ def mkdir(path, uid=None, gid=None, mode=None):
makedirs(path)
chrights(path, uid, gid, mode)
def chrights(path, uid=None, gid=None, mode=None, mtime=None):
def chrights(path, uid=None, gid=None, mode=None, mtime=None, strict=False):
'''
Set rights on a file
If strict is True, raise error if change right fail
'''
if uid is not None:
chown(path, uid, -1)
try:
chown(path, uid, -1)
except OSError:
if strict:
raise
if gid is not None:
chown(path, -1, gid)
try:
chown(path, -1, gid)
except OSError:
if strict:
raise
if mode is not None:
chmod(path, mode)
try:
chmod(path, mode)
except OSError:
if strict:
raise
if mtime is not None:
utime(path, (mtime, mtime))
try:
utime(path, (mtime, mtime))
except OSError:
if strict:
raise
def pathtype(path):
'''
......@@ -361,11 +378,12 @@ def pathsearch(name, path=None):
return join(abspath(d), name)
return None
def isfile(path):
def islocal(path, must_exists=False):
'''
Return True if path is of type file
if must_exists is True, also check is file exists
'''
return pathtype(path) == "file"
return pathtype(path) == "file" and (not must_exists or exists(path))
def abspath(path):
'''
......