Newer
Older
# -*- python -*-
# -*- coding: utf-8 -*-
# Started 10/05/2011 by Seblu <seblu@seblu.net>
'''
Repository stuff
'''
from installsystems.printer import *
from installsystems.tarball import Tarball
from installsystems.image import Image, PackageImage
from installsystems.database import Database
try:
self.db = Database(config.dbpath)
except:
self.db = None
def __getattribute__(self, name):
'''
Raise an error if repository is unavailable
Unavailable can be caused because db is not accessible or
because repository is not initialized
'''
db = object.__getattribute__(self, "db")
config = object.__getattribute__(self, "config")
# config and init are always accessible
if name in ("init", "config"):
return object.__getattribute__(self, name)
# if no db (not init or not accessible) raise error
if db is None:
raise Exception("Repository %s is not availabe" % config.name)
return object.__getattribute__(self, name)
raise Exception("Repository creation must be local")
# creating local directory
try:
if os.path.exists(config.path):
istools.mkdir(config.path, config.uid, config.gid, config.dmod)
except Exception as e:
raise Exception("Unable to create directory %s: %s" % (config.path, e))
d = Database.create(config.dbpath)
istools.chrights(config.dbpath, uid=config.uid,
gid=config.gid, mode=config.fmod)
# load database
self.db = Database(config.dbpath)
# create/update last file
self.update_last()
def update_last(self):
# check local repository
if istools.pathtype(self.config.path) != "file":
raise Exception("Repository addition must be local")
last_path = os.path.join(self.config.path, self.config.lastname)
open(last_path, "w").write("%s\n" % int(time.time()))
istools.chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
except Exception as e:
raise Exception("Update last file failed: %s" % e)
last_path = os.path.join(config.path, config.lastname)
return int(istools.uopen(last_path, "r").read().rstrip())
except Exception as e:
raise Exception("Read last file failed: %s" % e)
return 0
if delete is true, remove original files
# check local repository
if istools.pathtype(self.config.path) != "file":
raise Exception("Repository addition must be local")
# cannot add already existant image
if self.has(image.name, image.version):
raise Exception("Image already in database, delete first!")
for obj in [ image ] + image.payload.values():
dest = os.path.join(self.config.path, obj.md5)
basesrc = os.path.basename(obj.path)
arrow("Skipping %s: already exists" % basesrc, 1)
arrow("Adding %s (%s)" % (basesrc, obj.md5), 1)
istools.copy(obj.path, dest,
self.config.uid, self.config.gid, self.config.fmod)
# copy is done. create a image inside repo
r_image = PackageImage(os.path.join(self.config.path, image.md5),
# checking must be done with original md5
r_image.md5 = image.md5
# checking image and payload after copy
r_image.check("Check image and payload after copy")
arrow("Adding metadata")
self.db.begin()
# insert image information
arrow("Image", 1)
self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?)",
(image.md5,
image.name,
image.version,
image.date,
image.author,
image.description,
image.size,
))
# insert data informations
arrow("Payloads", 1)
for name, obj in image.payload.items():
self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
(obj.md5,
image.md5,
name,
obj.isdir,
obj.size,
))
# on commit
self.db.commit()
# removing orginal files
if delete:
arrow("Removing original files")
for obj in [ image ] + image.payload.values():
arrow(os.path.basename(obj.path), 1)
os.unlink(obj.path)
'''
Delete an image from repository
'''
# check local repository
if istools.pathtype(self.config.path) != "file":
raise Exception("Repository deletion must be local")
# get md5 of files related to images (exception is raised if not exists
md5s = self.getmd5(name, version)
# cleaning db (must be done before cleaning)
arrow("Cleaning database")
arrow("Remove payloads from database", 1)
self.db.begin()
for md5 in md5s[1:]:
self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?",
(md5, md5s[0])).fetchone()
arrow("Remove image from database", 1)
self.db.ask("DELETE FROM image WHERE md5 = ?",
(md5s[0],)).fetchone()
self.db.commit()
# Removing script image
arrow("Removing files from pool")
def show(self, verbose=False):
'''
List images in repository
'''
images = self.db.ask("SELECT md5, name, version, date,\
author, description, size FROM image ORDER BY name, version").fetchall()
for (image_md5, image_name, image_version, image_date, image_author,
image_description, image_size) in images:
out("#light##yellow#%s #reset#v%s" % (image_name, image_version))
out(" #yellow#Date:#reset# %s" % time.asctime(time.gmtime(image_date)))
out(" #yellow#Description:#reset# %s" % image_description)
out(" #yellow#Author:#reset# %s" % image_author)
out(" #yellow#MD5:#reset# %s" % image_md5)
payloads = self.db.ask("SELECT md5, name, size FROM payload\
WHERE image_md5 = ?", (image_md5,)).fetchall()
for payload_md5, payload_name, payload_size in payloads:
out(" #light##yellow#Payload:#reset# %s" % payload_name)
out(" #yellow#Size:#reset# %s" % (istools.human_size(payload_size)))
out(" #yellow#MD5:#reset# %s" % payload_md5)
def search(self, pattern):
'''
Search pattern in a repository
'''
images = self.db.ask("SELECT name, version, author, description\
FROM image\
WHERE name LIKE ? OR\
description LIKE ? OR\
author LIKE ?",
tuple( ["%%%s%%" % pattern ] * 3)
).fetchall()
for name, version, author, description in images:
arrow("%s v%s" % (name, version), 1)
out(" #yellow#Author:#reset# %s" % author)
out(" #yellow#Description:#reset# %s" % description)
def _remove_file(self, filename):
'''
Remove a filename from pool. Check if it's not needed by db before
'''
# check existance in table image
have = False
for table in ("image", "payload"):
have = have or self.db.ask("SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table,
(filename,)).fetchone() is not None
# if no reference, delete!
if not have:
arrow("%s, deleted" % filename)
os.unlink(os.path.join(self.config.path, filename))
else:
arrow("%s, skipped" % filename)
'''
Return the existance of a package
'''
return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None
Return an image from a name and version
# is no version take the last
if version is None:
version = self.last(name)
# get file md5 from db
r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
(name,version)).fetchone()
if r is None:
raise Exception("No such image %s version %s" % (name, version))
path = os.path.join(self.config.path, r[0])
debug("Getting %s v%s from %s" % (name, version, path))
pkg = PackageImage(path, md5name=True)
pkg.md5 = r[0]
return pkg
def getmd5(self, name, version):
'''
Return an image md5 and payload md5 from name and version. Order matter !
Image md5 will still be the first
'''
# get file md5 from db
a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1",
(name,version)).fetchone()
if a is None:
raise Exception("No such image %s version %s" % (name, version))
b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?",
(a[0],)).fetchall()
return [ a[0] ] + [ x[0] for x in b ]
'''
Return last version of name in repo or -1 if not found
'''
r = self.db.ask("SELECT version FROM image WHERE name = ? ORDER BY version DESC LIMIT 1", (name,)).fetchone()
# no row => no way
if r is None:
return -1
# return last
return r[0]
class RepositoryConfig(object):
# set default value for arguments
self.name = name
self.path = ""
self._dbpath = None
umask = os.umask(0)
os.umask(umask)
self._fmod = 0666 & ~umask
self._dmod = 0777 & ~umask
def __str__(self):
l = []
for a in ("name", "path", "dbpath", "lastpath", "uid", "gid", "fmod", "dmod"):
l.append("%s: %s" % (a, getattr(self, a)))
return os.linesep.join(l)
def __eq__(self, other):
return vars(self) == vars(other)
def __ne__(self, other):
return not (self == other)
def __contains__(self, key):
return key in self.__dict__
if self._lastpath is None:
return os.path.join(self.path, self.lastname)
return self._lastpath
if self._dbpath is None:
return os.path.join(self.path, self.dbname)
return self._dbpath
# dbpath must be local, sqlite3 requirment
if istools.pathtype(value) != "file":
raise ValueError("Database path must be local")
self._dbpath = os.path.abspath(value)
@property
def uid(self):
return self._uid
@uid.setter
def uid(self, value):
if not value.isdigit():
self._uid = pwd.getpwnam(value).pw_uid
self._uid = int(value)
@property
def gid(self):
return self._gid
@gid.setter
def gid(self, value):
if not value.isdigit():
self._gid = grp.getgrnam(value).gr_gid
else:
self._gid = int(value)
@property
def fmod(self):
return self._fmod
@fmod.setter
def fmod(self, value):
if value.isdigit():
self._fmod = int(value, 8)
else:
raise ValueError("File mode must be an integer")
@property
def dmod(self):
return self._dmod
@dmod.setter
def dmod(self, value):
if value.isdigit():
self._dmod = int(value, 8)
else:
raise ValueError("Directory mode must be an integer")
def update(self, *args, **kwargs):
All attribute must already exists
'''
# autoset parameter in cmdline
for k in kwargs:
if hasattr(self, k):
try:
setattr(self, k, kwargs[k])
except Exception as e:
warn("Unable to set config parameter %s in repository %s: %s" % (k, self.name, e))
class RepositoryManager(object):
'''
Manage multiple repostories
This call implement a cache and a manager for multiple repositories
'''
self.timeout = 3 if timeout is None else timeout
self.repos = []
self.tempfiles = []
if cache_path is None:
self.cache_path = None
debug("No repository cache")
if istools.pathtype(cache_path) != "file":
raise NotImplementedError("Repository cache must be local")
self.cache_path = os.path.abspath(cache_path)
# must_path is a list of directory which must exists
# create directory if not exists
if not os.path.exists(self.cache_path):
os.mkdir(self.cache_path)
# ensure directories are avaiblable
if not os.access(self.cache_path, os.W_OK | os.X_OK):
raise Exception("%s is not writable or executable" % self.cache_path)
debug("Repository cache is in %s" % self.cache_path)
def __del__(self):
# delete temporary files (used by db)
for f in self.tempfiles:
try:
debug("Removing temporary db file %s" % f)
def __len__(self):
'''
Return the number of repository registered
'''
return len(self.repos)
def __getitem__(self, key):
'''
Return a repostiory by its position in list
if type(key) == int:
return self.repos[key]
elif type(key) == str:
for repo in self.repos:
if repo.config.name == key:
return repo
raise Exception("No repository named: %s" % key)
else:
raise TypeError
# if path is local, no needs to create a cache
if istools.isfile(config.path):
debug("Registering direct repository %s (%s)" % (config.path, config.name))
self.repos.append(Repository(config))
else:
debug("Registering cached repository %s (%s)" % (config.path, config.name))
self.repos.append(self._cachify(config))
def _cachify(self, config):
'''
Return a config of a cached repository from an orignal config file
'''
# find destination file and load last info
if config.name is None or self.cache_path is None:
# this is a forced temporary repository or without name repo
tempfd, filedest = tempfile.mkstemp()
os.close(tempfd)
self.tempfiles.append(filedest)
filedest = os.path.join(self.cache_path, config.name)
# create file if not exists
if not os.path.exists(filedest):
open(filedest, "wb")
# get remote last value
rlast = int(istools.uopen(config.lastpath).read().strip())
# get local last value
llast = int(os.stat(filedest).st_mtime)
# if repo is out of date, download it
if rlast != llast:
istools.copy(config.dbpath, filedest,
uid=config.uid,
gid=config.gid,
mode=config.fmod,
timeout=self.timeout)
os.utime(filedest, (rlast, rlast))
config.dbpath = filedest
'''
Crawl all repo to get the most recent image
'''
# search last version if needed
if version is None:
lv = -1
for repo in self.repos:
lv = max(lv, repo.last(name))
if lv < 0:
raise Exception("Unable to find last version of %s" % name)
version = lv
# search image in repos
for repo in self.repos:
if repo.has(name, version):
return repo.get(name, version)
raise Exception("Unable to find %s v%s" % (name, version))
def show(self, verbose=False):
'''
Show repository inside manager
'''
for repo in self.repos:
repo.config.name
s = "#light##blue#%s#reset#"% repo.config.name
if verbose: