Newer
Older
# -*- python -*-
# -*- coding: utf-8 -*-
# Started 10/05/2011 by Seblu <seblu@seblu.net>
'''
Repository stuff
'''
from installsystems.printer import *
from installsystems.tarball import Tarball
from installsystems.image import Image, PackageImage
from installsystems.database import Database
# check local repository
if istools.pathtype(config.path) != "file":
raise Exception("Repository creation must be local")
# creating local directory
try:
if os.path.exists(config.path):
istools.mkdir(config.path, config.uid, config.gid, config.dmod)
except Exception as e:
raise Exception("Unable to create directory %s: %s" % (config.path, e))
dbpath = os.path.join(config.path, config.dbname)
istools.chrights(dbpath, uid=config.uid, gid=config.gid, mode=config.fmod)
# check local repository
if istools.pathtype(self.config.path) != "file":
raise Exception("Repository addition must be local")
last_path = os.path.join(self.config.path, self.config.lastname)
open(last_path, "w").write("%s\n" % int(time.time()))
istools.chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
except Exception as e:
raise Exception("Update last file failed: %s" % e)
last_path = os.path.join(config.path, config.lastname)
return int(istools.uopen(last_path, "r").read().rstrip())
except Exception as e:
raise Exception("Read last file failed: %s" % e)
return 0
# check local repository
if istools.pathtype(self.config.path) != "file":
raise Exception("Repository addition must be local")
# cannot add already existant image
if self.has(image.name, image.version):
raise Exception("Image already in database, delete first!")
for obj in [ image ] + image.payload.values():
dest = os.path.join(self.config.path, obj.md5)
basesrc = os.path.basename(obj.path)
arrow("Skipping %s: already exists" % basesrc, 1)
arrow("Adding %s (%s)" % (basesrc, obj.md5), 1)
istools.copy(obj.path, dest,
self.config.uid, self.config.gid, self.config.fmod)
# copy is done. create a image inside repo
r_image = PackageImage(os.path.join(self.config.path, image.md5),
# checking must be done with original md5
r_image.md5 = image.md5
# checking image and payload after copy
r_image.check("Check image and payload after copy")
arrow("Adding metadata")
self.db.begin()
# insert image information
arrow("Image", 1)
self.db.ask("INSERT INTO image values (?,?,?,?,?,?,?)",
(image.md5,
image.name,
image.version,
image.date,
image.author,
image.description,
image.size,
))
# insert data informations
arrow("Payloads", 1)
for name, obj in image.payload.items():
self.db.ask("INSERT INTO payload values (?,?,?,?,?)",
(obj.md5,
image.md5,
name,
obj.isdir,
obj.size,
))
# on commit
self.db.commit()
# removing orginal files
if delete:
arrow("Removing original files")
for obj in [ image ] + image.payload.values():
arrow(os.path.basename(obj.path), 1)
os.unlink(obj.path)
'''
Delete an image from repository
'''
# check local repository
if istools.pathtype(self.config.path) != "file":
raise Exception("Repository deletion must be local")
# get md5 of files related to images (exception is raised if not exists
md5s = self.getmd5(name, version)
# cleaning db (must be done before cleaning)
arrow("Cleaning database")
arrow("Remove payloads from database", 1)
self.db.begin()
for md5 in md5s[1:]:
self.db.ask("DELETE FROM payload WHERE md5 = ? AND image_md5 = ?",
(md5, md5s[0])).fetchone()
arrow("Remove image from database", 1)
self.db.ask("DELETE FROM image WHERE md5 = ?",
(md5s[0],)).fetchone()
self.db.commit()
# Removing script image
arrow("Removing files from pool")
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def show(self, verbose=False):
'''
List images in repository
'''
images = self.db.ask("SELECT md5, name, version, date,\
author, description, size FROM image").fetchall()
for image_md5, image_name, image_version, image_date, image_author,\
image_description, image_size in images:
out('Name : %s' % image_name)
out('Version : %s' % image_version)
out('Date : %s' % time.asctime(time.gmtime(image_date)))
if verbose:
out('Description : %s' % image_description)
out('Author : %s' % image_author)
out('MD5 : %s' % image_md5)
out('Payload :')
payloads = self.db.ask("SELECT md5, name, size FROM payload\
WHERE image_md5 = ?", (image_md5,)).fetchall()
for payload_md5, payload_name, payload_size in payloads:
out(' Name : %s' % payload_name)
out(' Size : %s' % (istools.human_size(payload_size)))
out(' MD5 : %s' % payload_md5)
out('')
out('')
def _remove_file(self, filename):
'''
Remove a filename from pool. Check if it's not needed by db before
'''
# check existance in table image
have = False
for table in ("image", "payload"):
have = have or self.db.ask("SELECT md5 FROM %s WHERE md5 = ? LIMIT 1" % table,
(filename,)).fetchone() is not None
# if no reference, delete!
if not have:
arrow("%s, deleted" % filename)
os.unlink(os.path.join(self.config.path, filename))
else:
arrow("%s, skipped" % filename)
'''
Return the existance of a package
'''
return self.db.ask("SELECT name,version FROM image WHERE name = ? AND version = ? LIMIT 1", (name,version)).fetchone() is not None
'''
return a image from a name and version
'''
# get file md5 from db
r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
(name,version)).fetchone()
if r is None:
raise Exception("No such image %s version %s" % name, version)
path = os.path.join(self.config.path, r[0])
debug("Getting %s v%s from %s" % (name, version, path))
pkg = PackageImage(path, md5name=True)
pkg.md5 = r[0]
return pkg
def getmd5(self, name, version):
'''
return a image md5 and payload md5 from name and version. Order matter !
image md5 will still be the first
'''
# get file md5 from db
a = self.db.ask("SELECT md5 FROM image WHERE name = ? AND version = ? LIMIT 1",
(name,version)).fetchone()
if a is None:
raise Exception("No such image %s version %s" % (name, version))
b = self.db.ask("SELECT md5 FROM payload WHERE image_md5 = ?",
(a[0],)).fetchall()
return [ a[0] ] + [ x[0] for x in b ]
'''
Return last version of name in repo or -1 if not found
'''
r = self.db.ask("SELECT version FROM image WHERE name = ? ORDER BY version DESC LIMIT 1", (name,)).fetchone()
# no row => no way
if r is None:
return -1
# return last
return r[0]
class RepositoryConfig(object):
# set default value for arguments
self.name = name
self.path = ""
self._dbpath = None
umask = os.umask(0)
os.umask(umask)
self._fmod = 0666 & ~umask
self._dmod = 0777 & ~umask
def __str__(self):
l = []
for a in ("name", "path", "dbpath", "lastpath", "uid", "gid", "fmod", "dmod"):
l.append("%s: %s" % (a, getattr(self, a)))
return os.linesep.join(l)
def __eq__(self, other):
return vars(self) == vars(other)
def __ne__(self, other):
return not (self == other)
def __contains__(self, key):
return key in self.__dict__
if self._lastpath is None:
return os.path.join(self.path, self.lastname)
return self._lastpath
if self._dbpath is None:
return os.path.join(self.path, self.dbname)
return self._dbpath
# dbpath must be local, sqlite3 requirment
if istools.pathtype(value) != "file":
raise ValueError("Database path must be local")
self._dbpath = os.path.abspath(value)
@property
def uid(self):
return self._uid
@uid.setter
def uid(self, value):
if not value.isdigit():
self._uid = pwd.getpwnam(value).pw_uid
self._uid = int(value)
@property
def gid(self):
return self._gid
@gid.setter
def gid(self, value):
if not value.isdigit():
self._gid = grp.getgrnam(value).gr_gid
else:
self._gid = int(value)
@property
def fmod(self):
return self._fmod
@fmod.setter
def fmod(self, value):
if value.isdigit():
self._fmod = int(value, 8)
else:
raise ValueError("File mode must be an integer")
@property
def dmod(self):
return self._dmod
@dmod.setter
def dmod(self, value):
if value.isdigit():
self._dmod = int(value, 8)
else:
raise ValueError("Directory mode must be an integer")
def update(self, *args, **kwargs):
All attribute must already exists
'''
# autoset parameter in cmdline
for k in kwargs:
if hasattr(self, k):
try:
setattr(self, k, kwargs[k])
except Exception as e:
warn("Unable to set config parameter %s in repository %s: %s" % (k, self.name, e))
class RepositoryManager(object):
'''
Manage multiple repostories
This call implement a cache and a manager for multiple repositories
'''
self.timeout = 3 if timeout is None else timeout
self.repos = []
self.tempfiles = []
if cache_path is None:
self.cache_path = None
debug("No repository cache")
if istools.pathtype(cache_path) != "file":
raise NotImplementedError("Repository cache must be local")
self.cache_path = os.path.abspath(cache_path)
# must_path is a list of directory which must exists
# create directory if not exists
if not os.path.exists(self.cache_path):
os.mkdir(self.cache_path)
# ensure directories are avaiblable
if not os.access(self.cache_path, os.W_OK | os.X_OK):
raise Exception("%s is not writable or executable" % self.cache_path)
debug("Repository cache is in %s" % self.cache_path)
def __del__(self):
# delete temporary files (used by db)
for f in self.tempfiles:
try:
debug("Removing temporaty db file %s" % f)
def __len__(self):
'''
Return the number of repository registered
'''
return len(self.repos)
def __getitem__(self, key):
'''
Return a repostiory by its possition in list
'''
return self.repos[key]
# if path is local, no needs to create a cache
if istools.isfile(config.path):
debug("Registering direct repository %s (%s)" % (config.path, config.name))
self.repos.append(Repository(config))
else:
debug("Registering cached repository %s (%s)" % (config.path, config.name))
self.repos.append(self._cachify(config))
def _cachify(self, config):
'''
Return a config of a cached repository from an orignal config file
'''
# find destination file and load last info
if config.name is None or self.cache_path is None:
# this is a forced temporary repository or without name repo
tempfd, filedest = tempfile.mkstemp()
os.close(tempfd)
self.tempfiles.append(filedest)
filedest = os.path.join(self.cache_path, config.name)
# create file if not exists
if not os.path.exists(filedest):
open(filedest, "wb")
# get remote last value
rlast = int(istools.uopen(config.lastpath).read().strip())
# get local last value
llast = int(os.stat(filedest).st_mtime)
# if repo is out of date, download it
if rlast != llast:
istools.copy(config.dbpath, filedest,
uid=config.uid,
gid=config.gid,
mode=config.fmod,
timeout=self.timeout)
os.utime(filedest, (rlast, rlast))
config.dbpath = filedest
'''
Crawl all repo to get the most recent image
'''
# search last version if needed
if version is None:
lv = -1
for repo in self.repos:
lv = max(lv, repo.last(name))
if lv < 0:
raise Exception("Unable to find last version of %s" % name)
version = lv
# search image in repos
for repo in self.repos:
if repo.has(name, version):
return repo.get(name, version)
raise Exception("Unable to find %s v%s" % (name, version))