diff --git a/bin/isinstall b/bin/isinstall
index b7008f35f56585e35d4376280d410a059cbac0bd..1ded51e28a135181419a729f289d8122ecf21823 100755
--- a/bin/isinstall
+++ b/bin/isinstall
@@ -13,7 +13,7 @@ import argparse
import installsystems
import installsystems.tools as istools
from installsystems.printer import *
-from installsystems.repository import RepositoryManager
+from installsystems.repository import RepositoryManager, RepositoryConfig
from installsystems.image import PackageImage
from installsystems.config import ConfigFile
@@ -34,12 +34,16 @@ p_main.add_argument('-d', "--debug", action=DebugAction, nargs=0,
help="active debug mode")
p_main.add_argument('-q', "--quiet", action="store_false", dest="verbose", default=True,
help="active quiet mode")
+p_main.add_argument("--no-cache", action="store_false", default=False,
+ help="Not use persistent db caching")
p_main.add_argument("-c", "--config", dest="config", type=str, default=None,
help="config file path")
p_main.add_argument("-v", "--image-version", dest="image_version", type=int, default=None,
help="image version")
p_main.add_argument("-t", "--timeout", dest="timeout", type=int, default=None,
help="download timeout")
+p_main.add_argument("-r", "--repo", dest="repos", action="append", default=[],
+ help="repository (can be specified more than one time)")
p_main.add_argument("image_name", type=str,
help="image to install (path or name)")
try:
@@ -52,10 +56,17 @@ try:
if image_name_type == "file":
pkg = PackageImage(istools.abspath(args.image_name))
elif image_name_type == "name":
+ # remove cache is asked
+ if args.no_cache:
+ config.cache = None
# init repo cache object
- repoman = RepositoryManager(timeout=args.timeout, verbose=args.verbose)
- # register repositories
- repoman.register(config.repos)
+ repoman = RepositoryManager(config.cache, timeout=args.timeout, verbose=args.verbose)
+ # register config repositories
+ for crepo in config.repos:
+ repoman.register(crepo)
+ # register command line repositories
+ for rpath in args.repos:
+ repoman.register(RepositoryConfig(None, path=rpath))
# get image package
pkg = repoman.get(args.image_name, args.image_version)
else:
diff --git a/bin/isrepo b/bin/isrepo
index a4ef2b9d25b75ed9e1d5b5501db3c5434766c563..73c23de4963e3de054599afce9dfb3e7cb1076d3 100755
--- a/bin/isrepo
+++ b/bin/isrepo
@@ -26,24 +26,23 @@ def init(args):
'''Create an empty fresh repo tree'''
# call init from library
try:
- Repository.create(args.repository, args.verbose)
+ Repository.create(args.repository, verbose=args.verbose)
except Exception as e:
raise Exception("init failed: %s" % e)
def add(args):
'''Add a package to repository'''
try:
- repo = Repository(args.repository, args.verbose)
- pkg = PackageImage(args.path, args.verbose)
+ repo = Repository(args.repository, verbose=args.verbose)
+ pkg = PackageImage(args.path, verbose=args.verbose)
repo.add(pkg)
except Exception as e:
- raise
raise Exception("add failed: %s" % e)
def delete(args):
'''Remove a package from repository'''
try:
- repo = Repository(args.repository, args.verbose)
+ repo = Repository(args.repository, verbose=args.verbose)
repo.delete(args.image_name, args.image_version)
except Exception as e:
raise Exception("del failed: %s" % e)
diff --git a/installsystems/config.py b/installsystems/config.py
index d32a01bcd922a853aaa64e7cd3e7c11646260d50..9b75f8fc2c806cded976d20adf2449942ba900ef 100644
--- a/installsystems/config.py
+++ b/installsystems/config.py
@@ -58,7 +58,7 @@ class ConfigFile(object):
def _cache_paths(self):
'''List all candidates to cache directories. Alive or not'''
- dirs = ["/var/tmp", "/tmp"]
+ dirs = [] # ["/var/tmp", "/tmp"]
# we have a different behaviour if we are root
dirs.insert(0, "/var/cache" if os.getuid() == 0 else os.path.expanduser("~/.cache"))
return map(lambda x: os.path.join(x, self.prefix), dirs)
diff --git a/installsystems/database.py b/installsystems/database.py
index fd62e276de64cf529a10750b43917843c3be8591..597f4248eb2a99638497241d6ba278d43469b65f 100644
--- a/installsystems/database.py
+++ b/installsystems/database.py
@@ -11,12 +11,16 @@ import os
import shutil
import tarfile
import cStringIO
+import sqlite3
import installsystems.tools as istools
+import installsystems.template as istemplate
from installsystems.tarball import Tarball
from installsystems.printer import *
class Database(object):
- '''Abstract repo database stuff'''
+ ''' Abstract repo database stuff
+ It needs to be local cause of sqlite3 which need to open a file
+ '''
db_format = "1"
@@ -26,23 +30,27 @@ class Database(object):
# check locality
if istools.pathtype(path) != "file":
raise NotImplementedError("Database creation must be local")
- dbpath = istools.abspath(path)
- if os.path.exists(dbpath):
+ path = os.path.abspath(path)
+ if os.path.exists(path):
raise Exception("Database already exists. Remove it before")
try:
- tarball = Tarball.open(dbpath, mode="w:gz", dereference=True)
- tarball.add_str("format", Database.db_format, tarfile.REGTYPE, 0444)
- tarball.close()
+ conn = sqlite3.connect(path, isolation_level=None)
+ conn.execute("PRAGMA foreign_keys = ON")
+ conn.executescript(istemplate.createdb)
+ conn.commit()
+ conn.close()
except Exception as e:
raise Exception("Create database failed: %s" % e)
return cls(path, verbose)
def __init__(self, path, verbose=True):
- self.path = istools.abspath(path)
+ # check locality
+ if istools.pathtype(path) != "file":
+ raise NotImplementedError("Database creation must be local")
+ self.path = os.path.abspath(path)
self.verbose = verbose
- # load db in memory
- self.file = cStringIO.StringIO()
- shutil.copyfileobj(istools.uopen(self.path), self.file)
+ self.conn = sqlite3.connect(self.path, isolation_level=None)
+ self.conn.execute("PRAGMA foreign_keys = ON")
def get(self, name, version):
'''Return a description dict from a package name'''
@@ -55,44 +63,48 @@ class Database(object):
except KeyError:
raise Exception("No package %s version %s in metadata" % (name, version))
except Exception as e:
- raise Exception("Unable to read db %s version %s: e" % (name, version, e))
+ raise Exception("Unable to read db %s version %s: %s" % (name, version, e))
# convert loaded data into dict (json parser)
try:
return json.loads(rdata)
except Exception as e:
raise Exception("Invalid metadata in package %s version %s: e" % (name, version, e))
+ def ask(self, sql, args=()):
+ '''Ask question to db'''
+ return self.conn.execute(sql, args)
+
def add(self, package):
'''Add a packaged image to a db'''
- arrow("Adding metadata to db", 1, self.verbose)
- # check locality
- if istools.pathtype(self.path) != "file":
- raise NotImplementedError("Database addition must be local")
- # naming
- newdb_path = "%s.new" % self.path
- # compute md5
- arrow("Formating metadata", 2, self.verbose)
- desc = package.description
- desc["md5"] = package.md5
- jdesc = json.dumps(desc)
try:
- arrow("Adding metadata", 2, self.verbose)
- self.file.seek(0)
- newfile = cStringIO.StringIO()
- db = Tarball.open(fileobj=self.file, mode='r:gz')
- newdb = Tarball.open(fileobj=newfile, mode='w:gz')
- for ti in db.getmembers():
- if ti.name != package.id:
- newdb.addfile(ti, db.extractfile(ti))
- newdb.add_str(package.id, jdesc, tarfile.REGTYPE, 0644)
- db.close()
- newdb.close()
- # writing to disk
- arrow("Writing to disk", 2, self.verbose)
- self.file.close()
- self.file = newfile
- self.write()
+ # let's go
+ arrow("Begin transaction to db", 1, self.verbose)
+ self.conn.execute("BEGIN TRANSACTION")
+ # insert image information
+ arrow("Add image metadata", 2, self.verbose)
+ self.conn.execute("INSERT OR REPLACE INTO image values (?,?,?,?,?,?,?)",
+ (package.md5,
+ package.name,
+ package.version,
+ package.date,
+ package.author,
+ package.description,
+ package.size,
+ ))
+ # insert data informations
+ arrow("Add data metadata", 2, self.verbose)
+ for key,value in package.data.items():
+ self.conn.execute("INSERT OR REPLACE INTO data values (?,?,?,?)",
+ (value["md5"],
+ package.md5,
+ key,
+ value["size"]
+ ))
+ # on commit
+ arrow("Commit transaction to db", 1, self.verbose)
+ self.conn.execute("COMMIT TRANSACTION")
except Exception as e:
+ raise
raise Exception("Adding metadata fail: %s" % e)
def delete(self, name, version):
@@ -148,14 +160,3 @@ class Database(object):
if int(version) not in candidates:
return None
return self.get(name, version)
-
- def write(self):
- '''Write current dabatase into its file'''
- if istools.pathtype(self.path) != "file":
- raise NotImplementedError("Database writing must be local")
- try:
- dest = open(self.path, "w")
- self.file.seek(0)
- shutil.copyfileobj(self.file, dest)
- except Exception as e:
- raise Exception("Unable to write database: %s" % e)
diff --git a/installsystems/image.py b/installsystems/image.py
index 1118f219585811d149cff1891efd2c7ace6f7f4f..5115b6c8b6233903ef3f2d653c572eb0b8762e7d 100644
--- a/installsystems/image.py
+++ b/installsystems/image.py
@@ -240,88 +240,94 @@ class SourceImage(Image):
class PackageImage(Image):
'''Packaged image manipulation class'''
- def __init__(self, path, verbose=True):
+ def __init__(self, path, md5name=False, verbose=True):
Image.__init__(self)
self.path = istools.abspath(path)
self.base_path = os.path.dirname(self.path)
self.verbose = verbose
+ # tarball are named by md5 and not by real name
+ self.md5name = md5name
# load image in memory
arrow("Loading tarball in memory", 1, verbose)
- self.file = cStringIO.StringIO()
+ memfile = cStringIO.StringIO()
fo = istools.uopen(self.path)
- shutil.copyfileobj(fo, self.file)
+ (self.size, self.md5) = istools.copyfileobj(fo, memfile)
fo.close()
# set tarball fo
- self.file.seek(0)
- self.tarball = Tarball.open(fileobj=self.file, mode='r:gz')
- self.parse()
- self._md5 = None
+ memfile.seek(0)
+ self._tarball = Tarball.open(fileobj=memfile, mode='r:gz')
+ self._metadata = self.read_metadata()
- @property
- def md5(self):
- '''Return md5sum of the current tarball'''
- if self._md5 is None:
- self._md5 = istools.md5sum(self.path)
- return self._md5
+ def __getattr__(self, name):
+ """Give direct access to description field"""
+ if name in self._metadata:
+ return self._metadata[name]
+ raise AttributeError
@property
def id(self):
'''Return image versionned name / id'''
- return "%s-%s" % (self.description["name"], self.description["version"])
-
- @property
- def name(self):
- '''Return image name'''
- return self.description["name"]
-
- @property
- def version(self):
- '''Return image version'''
- return self.description["version"]
+ return "%s-%s" % (self._metadata["name"], self._metadata["version"])
@property
def filename(self):
'''Return image filename'''
return "%s%s" % (self.id, self.extension)
- @property
- def datas(self):
- '''Create a dict of data tarballs'''
- d = dict()
- for dt in self.description["data"]:
- d[dt] = dict(self.description["data"][dt])
- d[dt]["filename"] = "%s-%s%s" % (self.id, dt, self.extension_data)
- d[dt]["path"] = os.path.join(self.base_path, d[dt]["filename"])
- return d
-
- def parse(self):
- '''Parse tarball and extract metadata'''
+ def read_metadata(self):
+ '''Parse tarball and return metadata dict'''
# extract metadata
arrow("Read tarball metadata", 1, self.verbose)
- img_format = self.tarball.get_str("format")
- img_desc = self.tarball.get_str("description.json")
+ img_format = self._tarball.get_str("format")
+ img_desc = self._tarball.get_str("description.json")
# check format
- arrow("Read format", 2, self.verbose)
+ arrow("Read format file", 2, self.verbose)
if img_format != self.format:
raise Exception("Invalid tarball image format")
# check description
- arrow("Read description", 2, self.verbose)
+ arrow("Read description file", 2, self.verbose)
try:
- self.description = json.loads(img_desc)
+ desc = json.loads(img_desc)
except Exception as e:
raise Exception("Invalid description: %s" % e1)
+ # FIXME: we should check valid information here
+ return desc
- def check_md5(self):
- '''Check if md5 of data tarballs are correct'''
- arrow("Check MD5", 1, self.verbose)
- databalls = self.description["data"]
- for databall in databalls:
- md5_path = os.path.join(self.base_path, databall)
- arrow(os.path.relpath(md5_path), 2, self.verbose)
- md5_meta = databalls[databall]["md5"]
- md5_file = istools.md5sum(md5_path)
- if md5_meta != md5_file:
- raise Exception("Invalid md5: %s" % databall)
+ @property
+ def tarballs(self):
+ '''List path of all related tarballs'''
+ d_d = {}
+ name = os.path.join(self.base_path, self.md5) if self.md5name else self.path
+ d_d[name] = {"md5": self.md5, "size": self.size}
+ for key, value in self._metadata["data"].items():
+ if self.md5name:
+ name = os.path.join(self.base_path, value["md5"])
+ else:
+ name = os.path.join(self.base_path,
+ "%s-%s%s" % (self.id, key, self.extension_data))
+ d_d[name] = {"md5": value["md5"], "size": value["size"]}
+ return d_d
+
+ def tarcheck(self, message="Check MD5"):
+ '''Check md5 and size of tarballs are correct'''
+ arrow(message, 1, self.verbose)
+ # open /dev/null
+ dn = open("/dev/null", "w")
+ for key,value in self.tarballs.items():
+ arrow(os.path.basename(key), 2, self.verbose)
+ # open tarball
+ tfo = istools.uopen(key)
+ # compute sum and md5 using copy function
+ size, md5 = istools.copyfileobj(tfo ,dn)
+ # close tarball fo
+ tfo.close()
+ # check md5
+ if md5 != value["md5"]:
+ raise Exception("Invalid md5: %s" % key)
+ # check size
+ if size != value["size"]:
+ raise Exception("Invalid size: %s" % key)
+ dn.close()
def run_parser(self, gl):
'''Run parser scripts'''
@@ -336,14 +342,14 @@ class PackageImage(Image):
'''Run scripts in a tarball directory'''
arrow("Run %s" % directory, 1, self.verbose)
# get list of parser scripts
- l_scripts = self.tarball.getnames("%s/.*\.py" % directory)
+ l_scripts = self._tarball.getnames("%s/.*\.py" % directory)
# order matter!
l_scripts.sort()
# run scripts
for n_scripts in l_scripts:
arrow(os.path.basename(n_scripts), 2, self.verbose)
try:
- s_scripts = self.tarball.get_str(n_scripts)
+ s_scripts = self._tarball.get_str(n_scripts)
except Exception as e:
raise Exception("Extracting script %s fail: %s" %
(os.path.basename(n_scripts), e))
@@ -357,10 +363,10 @@ class PackageImage(Image):
def extractdata(self, dataname, target, filelist=None):
'''Extract a data tarball into target'''
# check if dataname exists
- if dataname not in self.description["data"].keys():
- raise Exception("No such data")
+ if dataname not in self._metadata["data"].keys():
+ raise Exception("No such data: %s" % dataname)
# tarball info
- tinfo = self.description["data"][dataname]
+ tinfo = self._metadata["data"][dataname]
# build data tar paths
paths = [ os.path.join(self.base_path, tinfo["md5"]),
os.path.join(self.base_path, "%s-%s%s" % (self.id,
diff --git a/installsystems/repository.py b/installsystems/repository.py
index 37a5b5053b5ea311527c2979a89a01ab6504c972..8f9d41953268cc7823b2743f76ea5c9ecc8ff881 100644
--- a/installsystems/repository.py
+++ b/installsystems/repository.py
@@ -12,6 +12,7 @@ import shutil
import pwd
import grp
import copy
+import tempfile
import installsystems
import installsystems.tools as istools
from installsystems.printer import *
@@ -20,12 +21,13 @@ from installsystems.image import Image, PackageImage
from installsystems.database import Database
class Repository(object):
- '''Repository class'''
+ ''' Repository class
+ '''
def __init__(self, config, verbose=True):
self.verbose = verbose
self.config = config
- self.db = Database(os.path.join(config.path, config.dbname), verbose=self.verbose)
+ self.db = Database(config.dbpath, verbose=self.verbose)
@classmethod
def create(cls, config, verbose=True):
@@ -40,16 +42,15 @@ class Repository(object):
if os.path.exists(config.path):
arrow("%s already exists" % config.path, 2, verbose)
else:
- istools.mkdir(config.path, config.chown, config.chgroup, config.dchmod)
+ istools.mkdir(config.path, config.uid, config.gid, config.dmod)
arrow("%s directory created" % config.path, 2, verbose)
except Exception as e:
raise Exception("Unable to create directory %s: %s" % (config.path, e))
# create database
dbpath = os.path.join(config.path, config.dbname)
d = Database.create(dbpath, verbose=verbose)
- istools.chrights(dbpath, config.chown, config.chgroup, config.fchmod)
+ istools.chrights(dbpath, uid=config.uid, gid=config.gid, mode=config.fmod)
# create last file
- arrow("Creating last file", 1, verbose)
self = cls(config, verbose)
self.update_last()
return self
@@ -63,8 +64,7 @@ class Repository(object):
arrow("Updating last file", 1, self.verbose)
last_path = os.path.join(self.config.path, self.config.lastname)
open(last_path, "w").write("%s\n" % int(time.time()))
- os.chown(last_path, self.config.chown, self.config.chgroup)
- os.chmod(last_path, self.config.fchmod)
+ istools.chrights(last_path, self.config.uid, self.config.gid, self.config.fmod)
except Exception as e:
raise Exception("Update last file failed: %s" % e)
@@ -82,37 +82,25 @@ class Repository(object):
# check local repository
if istools.pathtype(self.config.path) != "file":
raise NotImplementedError("Repository addition must be local")
- # copy file to directory
- arrow("MD5summing tarballs", 1, self.verbose)
- # build dict of file to add
- filelist = dict()
- # script tarball
- arrow(package.filename, 2, self.verbose)
- filelist[package.md5] = package.path
- # data tarballs
- datas = package.datas
- for dt in datas:
- dt_path = datas[dt]["path"]
- old_md5 = datas[dt]["md5"]
- arrow(os.path.relpath(dt_path), 2, self.verbose)
- md5 = istools.md5sum(dt_path)
- if md5 != old_md5:
- raise Exception("MD5 mismatch on %s" % dt_path)
- filelist[md5] = dt_path
+ # checking data tarballs md5 before copy
+ package.tarcheck("Check tarballs before copy")
# adding file to repository
- arrow("Adding files to directory", 1, self.verbose)
- for md5 in filelist:
- dest = os.path.join(self.config.path, md5)
- source = filelist[md5]
+ arrow("Copying files", 1, self.verbose)
+ for src,value in package.tarballs.items():
+ dest = os.path.join(self.config.path, value["md5"])
+ basesrc = os.path.basename(src)
if os.path.exists(dest):
- arrow("Skipping %s: already exists" % (os.path.basename(source)),
- 2, self.verbose)
+ arrow("Skipping %s: already exists" % basesrc, 2, self.verbose)
else:
- arrow("Adding %s (%s)" % (os.path.basename(source), md5), 2, self.verbose)
- istools.copy(source, dest,
- self.config.chown, self.config.chgroup, self.config.fchmod)
+ arrow("Adding %s (%s)" % (basesrc, value["md5"]), 2, self.verbose)
+ istools.copy(src, dest, self.config.uid, self.config.gid, self.config.fmod)
+ # copy is done. create a package inside repo
+ r_package = PackageImage(os.path.join(self.config.path, package.md5),
+ md5name=True, verbose=self.verbose)
+ # checking data tarballs md5 after copy
+ r_package.tarcheck("Check tarballs after copy")
# add description to db
- self.db.add(package)
+ self.db.add(r_package)
# update last file
self.update_last()
@@ -145,62 +133,55 @@ class Repository(object):
arrow("Updating last file", 1, self.verbose)
self.update_last()
+ def has(self, name, version):
+ return self.db.ask("select name,version from image where name = ? and version = ? limit 1", (name,version)).fetchone() is not None
+
def get(self, name, version):
'''return a package from a name and version of pakage'''
- desc = self.db.get(name, version)
- p = PackageImage(os.path.join(self.config.path, desc["md5"]), verbose=self.verbose)
- if p.md5 != desc["md5"]:
- raise Exception("Invalid package MD5")
- return p
+ debug("Getting %s v%s" % (name, version))
+ # get file md5 from db
+ r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
+ (name,version)).fetchone()
+ if r is None:
+ raise Exception("No such image %s version %s" % name, version)
+ return PackageImage(os.path.join(self.config.path, r[0]),
+ md5name=True,
+ verbose=self.verbose)
+
+ def last(self, name):
+ '''Return last version of name in repo or -1 if not found'''
+ r = self.db.ask("select version from image where name = ? order by version desc limit 1", (name,)).fetchone()
+ # no row => no way
+ if r is None:
+ return -1
+ # return last
+ return r[0]
+
class RepositoryConfig(object):
'''Repository configuration container'''
- def __init__(self, *args, **kwargs):
+ def __init__(self, name, **kwargs):
# set default value for arguments
- self.name = args[0]
+ self.name = name
+ self.path = ""
+ self._dbpath = None
self.dbname = "db"
+ self._lastpath = None
self.lastname = "last"
- self.path = ""
- self.chown = os.getuid()
- self.chgroup = os.getgid()
+ self._uid = os.getuid()
+ self._gid = os.getgid()
umask = os.umask(0)
os.umask(umask)
- self.fchmod = 0666 & ~umask
- self.dchmod = 0777 & ~umask
+ self._fmod = 0666 & ~umask
+ self._dmod = 0777 & ~umask
self.update(**kwargs)
- def update(self, *args, **kwargs):
- '''Update attribute with checking value
- All attribute must already exists
- '''
- # autoset parameter in cmdline
- for k in kwargs:
- if hasattr(self, k):
- # attribute which are not in the following list cannot be loaded
- # from configuration
- try:
- # convert userid
- if k == "chown":
- if not k.isdigit():
- kwargs[k] = pwd.getpwnam(kwargs[k]).pw_uid
- setattr(self, k, int(kwargs[k]))
- # convert gid
- elif k == "chgroup":
- if not k.isdigit():
- kwargs[k] = grp.getgrnam(kwargs[k]).gr_gid
- setattr(self, k, int(kwargs[k]))
- # convert file mode
- elif k in ("fchmod", "dchmod"):
- setattr(self, k, int(kwargs[k], 8))
- # convert repo path
- elif k in ("image", "data"):
- setattr(self, k, istools.abspath(kwargs[k]))
- # else is string
- else:
- setattr(self, k, kwargs[k])
- except Exception as e:
- warn("Unable to set config parameter %s in repository %s: %s" % (k, self.name, e))
+ def __str__(self):
+ l = []
+ for a in ("name", "path", "dbpath", "lastpath", "uid", "gid", "fmod", "dmod"):
+ l.append("%s: %s" % (a, getattr(self, a)))
+ return os.linesep.join(l)
def __eq__(self, other):
return vars(self) == vars(other)
@@ -211,150 +192,179 @@ class RepositoryConfig(object):
def __contains__(self, key):
return key in self.__dict__
-class RepositoryManager(object):
- '''Manage multiple repostories'''
+ @property
+ def lastpath(self):
+ """return the last file complete path"""
+ if self._lastpath is None:
+ return os.path.join(self.path, self.lastname)
+ return self._lastpath
- def __init__(self, timeout=None, verbose=True):
- self.verbose = verbose
- self.timeout = 3 if timeout is None else timeout
- self.repos = {}
+ @lastpath.setter
+ def lastpath(self, value):
+ '''Set last path'''
+ self._lastpath = value
- def register(self, configs):
- '''Register a list of repository from its config'''
- for conf in configs:
- self.repos[conf.name] = Repository(conf, self.verbose)
+ @property
+ def dbpath(self):
+ """return the db complete path"""
+ if self._dbpath is None:
+ return os.path.join(self.path, self.dbname)
+ return self._dbpath
- def find_image(self, name, version):
- '''Find a repository containing image'''
- if version is None:
- arrow("Searching last version of %s" % name, 1, self.verbose)
+ @dbpath.setter
+ def dbpath(self, value):
+ '''Set db path'''
+ # dbpath must be local, sqlite3 requirment
+ if istools.pathtype(value) != "file":
+ raise ValueError("Database path must be local")
+ self._dbpath = os.path.abspath(value)
+
+ @property
+ def uid(self):
+ '''Return owner of repository'''
+ return self._uid
+
+ @uid.setter
+ def uid(self, value):
+ '''Define user name owning repository'''
+ if not value.isdigit():
+ self._uid = pwd.getpwnam(value).pw_uid
else:
- arrow("Searching %s version %s " % (name, version), 1, self.verbose)
- img = None
- # search in all repositories
- desc = None
- for repo in self.repos:
- desc = self.repos[repo].db.find(name, version)
- if desc is not None:
- # \o/
- break
- if desc is None:
- arrow("Not found", 2, self.verbose)
- if version is None:
- error("Unable to find a version of image %s" % name)
+ self._uid = int(value)
+
+ @property
+ def gid(self):
+ '''Return group of the repository'''
+ return self._gid
+
+ @gid.setter
+ def gid(self, value):
+ '''Define group owning repository'''
+ if not value.isdigit():
+ self._gid = grp.getgrnam(value).gr_gid
+ else:
+ self._gid = int(value)
+
+ @property
+ def fmod(self):
+ '''Return new file mode'''
+ return self._fmod
+
+ @fmod.setter
+ def fmod(self, value):
+ '''Define new file mode'''
+ if value.isdigit():
+ self._fmod = int(value, 8)
+ else:
+ raise ValueError("File mode must be an integer")
+
+ @property
+ def dmod(self):
+ '''Return new directory mode'''
+ return self._dmod
+
+ @dmod.setter
+ def dmod(self, value):
+ '''Define new directory mode'''
+ if value.isdigit():
+ self._dmod = int(value, 8)
+ else:
+ raise ValueError("Directory mode must be an integer")
+
+ def update(self, *args, **kwargs):
+ '''Update attribute with checking value
+ All attribute must already exists
+ '''
+ # autoset parameter in cmdline
+ for k in kwargs:
+ if hasattr(self, k):
+ try:
+ setattr(self, k, kwargs[k])
+ except Exception as e:
+ warn("Unable to set config parameter %s in repository %s: %s" % (k, self.name, e))
else:
- error("Unable to find image %s version %s" % (name, version))
- arrow("Found %s version %s " % (desc["name"], desc["version"]), 2, self.verbose)
- return (desc, self.repos[repo])
+ debug("No such repository parameter: %s" % k)
- def get(self, name, version=None):
- '''Return a package object from local cache'''
- # find an image name/version in repository
- (desc, repo) = self.find_image(name, version)
- # get pkg object
- return repo.get(desc["name"], desc["version"])
-class RepositoryCache(object):
- '''Local repository cache class'''
+class RepositoryManager(object):
+ '''
+ Manage multiple repostories
+
+ This call implement a cache and a manager for multiple repositories
+ '''
- def __init__(self, cache_path, timeout=3, verbose=True):
+ def __init__(self, cache_path=None, timeout=None, verbose=True):
self.verbose = verbose
- self.timeout = timeout
- self.repos = {}
- self.path = os.path.abspath(cache_path)
- # ensure cache directories are avaiblable
- if not os.path.exists(self.path):
- os.mkdir(self.path)
- if not os.access(self.path, os.W_OK | os.X_OK):
- raise Exception("%s is not writable or executable" % path)
- debug("Repository cache is in %s" % self.path)
-
- def register(self, configs):
- '''Register a list of repository from its config'''
- for conf in configs:
- self.repos[conf.name] = {}
- # keep original repository conf
- self.repos[conf.name]["orig"] = Repository(conf, self.verbose)
- # change configuration to make remote repository in cache
- cconf = copy.copy(conf)
- cconf.image = os.path.join(self.path, conf.name)
- cconf.data = "/dev/null"
- self.repos[conf.name]["cache"] = Repository(cconf, self.verbose)
- # create a local directory
- if not os.path.exists(cconf.image):
- os.mkdir(cconf.image)
-
- def update(self):
- '''Update cache info'''
- arrow("Updating repositories", 1, self.verbose)
- for r in self.repos:
- # last local
- local_last = self.last(r)
- # copy last file
- arrow("Checking %s repository last" % r, 2, self.verbose)
- istools.copy(self.repos[r]["orig"].last_path,
- self.repos[r]["cache"].last_path, timeout=self.timeout)
- # last after update
- remote_last = self.last(r)
- debug("%s: last: local: %s, remote:%s" % (r, local_last, remote_last))
- # Updating db?
- remote_db = self.repos[r]["orig"].db.path
- local_db = self.repos[r]["cache"].db.path
- if remote_last > local_last or not os.path.exists(local_db):
- # copy db file
- arrow("Copying %s repository db" % r, 2, self.verbose)
- istools.copy(remote_db, local_db, timeout=self.timeout)
-
- def last(self, reponame):
- '''Return the last timestamp of a repo'''
- last_path = os.path.join(self.path, reponame, "last")
- try:
- return int(open(last_path, "r").read().rstrip())
- except Exception:
- return 0
-
- def get_image(self, reponame, imagename, imageversion):
- '''Obtain a local path in cache for a remote image in repo'''
- arrow("Getting image", 1, self.verbose)
- filename = "%s-%s%s" % (imagename, imageversion, Image.image_extension)
- cachepath = os.path.join(self.repos[reponame]["cache"].config.image, filename)
- # return db path if exists
- if os.path.exists(cachepath):
- arrow("Found in cache", 2, self.verbose)
+ self.timeout = 3 if timeout is None else timeout
+ self.repos = []
+ self.tempfiles = []
+ if cache_path is None:
+ self.cache_path = None
+ debug("No repository cache")
else:
- # get remote image
- remotepath = os.path.join(self.repos[reponame]["orig"].config.image, filename)
- arrow("Copying from repository", 2, self.verbose)
- istools.copy(remotepath, cachepath, timeout=self.timeout)
- return cachepath
-
- def find_image(self, name, version):
- '''Find an image in repositories'''
- if version is None:
- arrow("Searching last version of %s" % name, 1, self.verbose)
+ if istools.pathtype(cache_path) != "file":
+ raise NotImplementedError("Repository cache must be local")
+ self.cache_path = os.path.abspath(cache_path)
+ # must_path is a list of directory which must exists
+ # create directory if not exists
+ if not os.path.exists(self.cache_path):
+ os.mkdir(self.cache_path)
+ # ensure directories are avaiblable
+ if not os.access(self.cache_path, os.W_OK | os.X_OK):
+ raise Exception("%s is not writable or executable" % t_path)
+ debug("Repository cache is in %s" % self.cache_path)
+
+ def __del__(self):
+ # delete temporary files (used by db)
+ for f in self.tempfiles:
+ try:
+ debug("Removing %s" % f)
+ os.unlink(f)
+ except OSError:
+ pass
+
+ def register(self, config):
+ '''Register a repository from its config'''
+ debug("Registering repository %s (%s)" % (config.path, config.name))
+ # find destination file and load last info
+ if config.name is None or self.cache_path is None:
+ # this is a forced temporary repository or without name repo
+ tempfd, filedest = tempfile.mkstemp()
+ os.close(tempfd)
+ self.tempfiles.append(filedest)
else:
- arrow("Searching %s version %s " % (name, version), 1, self.verbose)
- img = None
- # search in all repositories
- for repo in self.repos:
- tempdb = Database(self.repos[repo]["cache"].db.path, False)
- img = tempdb.find(name, version)
- if img is not None:
- # \o/
- break
- if img is None:
- arrow("Not found", 2, self.verbose)
- if version is None:
- error("Unable to find a version of image %s" % name)
- else:
- error("Unable to find image %s version %s" % (name, version))
- arrow("Found %s version %s " % (img[0], img[1]), 2, self.verbose)
- return (repo, img[0], img[1])
+ filedest = os.path.join(self.cache_path, config.name)
+ # create file if not exists
+ if not os.path.exists(filedest):
+ open(filedest, "wb")
+ # get remote last value
+ rlast = int(istools.uopen(config.lastpath).read().strip())
+ # get local last value
+ llast = int(os.stat(filedest).st_mtime)
+ # if repo is out of date, download it
+ if rlast != llast:
+ arrow("Getting %s" % config.dbpath, 1, self.verbose)
+ istools.copy(config.dbpath, filedest,
+ uid=config.uid,
+ gid=config.gid,
+ mode=config.fmod,
+ timeout=self.timeout)
+ os.utime(filedest, (rlast, rlast))
+ config.dbpath = filedest
+ self.repos.append(Repository(config, self.verbose))
def get(self, name, version=None):
- '''Return a package object from local cache'''
- r, n, v = self.find_image(name, version)
- # download image if not in cache
- path = self.get_image(r, n, v)
- # create an object image
- return PackageImage(path, self.verbose)
+ '''Crawl all repo to get the most recent image'''
+ # search last version if needed
+ if version is None:
+ lv = -1
+ for repo in self.repos:
+ lv = max(lv, repo.last(name))
+ if lv < 0:
+ raise Exception("Unable to find last version of %s" % name)
+ version = lv
+ # search image in repos
+ for repo in self.repos:
+ if repo.has(name, version):
+ return repo.get(name, version)
+ raise Exception("Unable to find %s v%s" % (name, version))
diff --git a/installsystems/template.py b/installsystems/template.py
index 61f57aaa3d44c93516404e0ce83cae22fe906215..748ffd774e02698d1cb51573654e337b890dece8 100644
--- a/installsystems/template.py
+++ b/installsystems/template.py
@@ -28,3 +28,20 @@ image.extractdata("rootfs", args.target)
# vim:set ts=2 sw=2 noet:
"""
+
+createdb = """
+CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
+ name TEXT NOT NULL,
+ version INTEGER NOT NULL,
+ date INTEGER NOT NULL,
+ author TEXT,
+ description TEXT,
+ size INTEGER NOT NULL,
+ UNIQUE(name, version));
+
+CREATE TABLE data (md5 TEXT NOT NULL,
+ image_md5 TEXT NOT NULL REFERENCES image(md5),
+ name TEXT NOT NULL,
+ size INTEGER NOT NULL,
+ PRIMARY KEY(md5, image_md5));
+"""
diff --git a/installsystems/tools.py b/installsystems/tools.py
index 7a3fb8258d4e0d6d016417cfb3acfc1cb7ab74b0..f42f85c78a1cfe3bc012c25d0788d0ca970b9138 100644
--- a/installsystems/tools.py
+++ b/installsystems/tools.py
@@ -26,6 +26,20 @@ def md5sum(path=None, fileobj=None):
m.update(buf)
return m.hexdigest()
+def copyfileobj(sfile, dfile):
+ """Copy data from sfile to dfile coputing length and md5 on the fly"""
+ f_sum = hashlib.md5()
+ f_len = 0
+ while True:
+ buf = sfile.read(1024 * f_sum.block_size)
+ buf_len = len(buf)
+ if buf_len == 0:
+ break
+ f_len += buf_len
+ f_sum.update(buf)
+ dfile.write(buf)
+ return (f_len , f_sum.hexdigest())
+
def copy(source, destination, uid=None, gid=None, mode=None, timeout=None):
'''Copy a source to destination. Take care of path type'''
stype = pathtype(source)