From 26aaa96f5aba2f6fad1e7e253b83893e94afc7e0 Mon Sep 17 00:00:00 2001
From: Seblu <sebastien.luttringer@smartjog.com>
Date: Tue, 31 May 2011 20:16:26 +0200
Subject: [PATCH] Improve remote cache. Http transport enabled
---
bin/isinstall | 4 +-
installsystems/config.py | 9 ++--
installsystems/database.py | 4 +-
installsystems/repository.py | 87 +++++++++++++++++++++---------------
installsystems/tools.py | 13 ++++--
5 files changed, 70 insertions(+), 47 deletions(-)
diff --git a/bin/isinstall b/bin/isinstall
index c1ec467..687fdb5 100755
--- a/bin/isinstall
+++ b/bin/isinstall
@@ -45,9 +45,9 @@ try:
# load config
config = ConfigFile("isinstall", args.config)
# looks if arguments is a file or image name
- image_name_type = istools.get_path_type(args.image_name)
+ image_name_type = istools.pathtype(args.image_name)
if image_name_type == "file":
- pkg = PackageImage(istools.complete_path(args.image_name))
+ pkg = PackageImage(istools.abspath(args.image_name))
elif image_name_type == "name":
# init repo cache object
repocache = RepositoryCache(config.cache, verbose=args.verbose)
diff --git a/installsystems/config.py b/installsystems/config.py
index 087cacf..d2d212c 100644
--- a/installsystems/config.py
+++ b/installsystems/config.py
@@ -32,7 +32,7 @@ class ConfigFile(object):
'''Load/Reload config file'''
# seting default config
self._config = {}
- self._repos = {}
+ self._repos = []
# loading config file if exists
if self.path is not None:
debug("Loading config file: %s" % self.path)
@@ -49,7 +49,7 @@ class ConfigFile(object):
if "image" not in cp.options(rep):
continue
# get all options in repo
- self._repos[rep] = RepositoryConfig(rep, **dict(cp.items(rep)))
+ self._repos.append( RepositoryConfig(rep, **dict(cp.items(rep))))
except Exception as e:
raise
raise Exception("Unable load file %s: %s" % (self.path, e))
@@ -89,5 +89,6 @@ class ConfigFile(object):
@property
def repos(self):
- '''Get a list of repository available'''
- return self._repos.copy()
+ '''Get a dict of repository available'''
+ # deep copy
+ return list(self._repos)
diff --git a/installsystems/database.py b/installsystems/database.py
index 2237134..6a7667e 100644
--- a/installsystems/database.py
+++ b/installsystems/database.py
@@ -22,7 +22,7 @@ class Database(object):
@classmethod
def create(cls, path, verbose=True):
arrow("Creating repository database", 1, verbose)
- dbpath = os.path.abspath(path)
+ dbpath = istools.abspath(path)
if os.path.exists(dbpath):
raise Exception("db already exists")
try:
@@ -34,7 +34,7 @@ class Database(object):
return cls(path, verbose)
def __init__(self, path, verbose=True):
- self.path = os.path.abspath(path)
+ self.path = istools.abspath(path)
self.verbose = verbose
def add(self, package):
diff --git a/installsystems/repository.py b/installsystems/repository.py
index 88fc9ac..d03ed39 100644
--- a/installsystems/repository.py
+++ b/installsystems/repository.py
@@ -11,6 +11,7 @@ import time
import shutil
import pwd
import grp
+import copy
import installsystems
import installsystems.tools as istools
from installsystems.printer import *
@@ -176,61 +177,75 @@ class RepositoryCache(object):
'''Local repository cache class'''
def __init__(self, cache_path, verbose=True):
- self.base_path = os.path.abspath(cache_path)
- self.image_path = os.path.join(self.base_path, "image")
- self.last_path = os.path.join(self.base_path, "last")
- self.db_path = os.path.join(self.base_path, "db")
- for path in (self.base_path, self.image_path, self.last_path, self.db_path):
- if not os.path.exists(path):
- os.mkdir(path)
- if not os.access(path, os.W_OK | os.X_OK):
- raise Exception("%s is not writable or executable" % path)
self.verbose = verbose
- self.repos = dict()
- debug("Repository cache is in %s" % self.base_path)
+ self.repos = {}
+ self.path = os.path.abspath(cache_path)
+ # ensure cache directories are avaiblable
+ if not os.path.exists(self.path):
+ os.mkdir(self.path)
+ if not os.access(self.path, os.W_OK | os.X_OK):
+ raise Exception("%s is not writable or executable" % path)
+ debug("Repository cache is in %s" % self.path)
- def register(self, iterepo):
- '''Register a repository to track'''
- for r in iterepo:
- self.repos[r[0]] = Repository(istools.abspath(r[1]),
- istools.abstpath(r[2]),
- verbose=self.verbose)
+ def register(self, configs):
+ '''Register a list of repository from its config'''
+ for conf in configs:
+ self.repos[conf.name] = {}
+ # keep original repository conf
+ self.repos[conf.name]["orig"] = Repository(conf, self.verbose)
+ # change configuration to make remote repository in cache
+ cconf = copy.copy(conf)
+ cconf.image = os.path.join(self.path, conf.name)
+ cconf.data = "/dev/null"
+ self.repos[conf.name]["cache"] = Repository(cconf, self.verbose)
+ # create a local directory
+ if not os.path.exists(cconf.image):
+ os.mkdir(cconf.image)
def update(self):
'''Update cache info'''
arrow("Updating repositories", 1, self.verbose)
for r in self.repos:
- debug("%s: remote_last: %s, local_last:%s" % (r,
- self.repos[r].last(),
- self.last(r)))
- if self.repos[r].last() > self.last(r):
- # copy last file
- istools.copy(self.repos[r].last_path, os.path.join(self.last_path, r))
+ # last local
+ local_last = self.last(r)
+ # copy last file
+ arrow("Copying %s repository last" % r, 2, self.verbose)
+ istools.copy(self.repos[r]["orig"].last_path,
+ self.repos[r]["cache"].last_path,)
+ # last after update
+ remote_last = self.last(r)
+ debug("%s: last: local: %s, remote:%s" % (r, local_last, remote_last))
+ # Updating db?
+ remote_db = self.repos[r]["orig"].db.path
+ local_db = self.repos[r]["cache"].db.path
+ if remote_last > local_last or not os.path.exists(local_db):
# copy db file
- istools.copy(self.repos[r].db.path, os.path.join(self.db_path, r))
+ arrow("Copying %s repository db" % r, 2, self.verbose)
+ istools.copy(remote_db, local_db)
arrow("%s updated" % r, 2, self.verbose)
def last(self, reponame):
'''Return the last timestamp of a repo'''
- last_path = os.path.join(self.last_path, reponame)
- if os.path.exists(last_path):
+ last_path = os.path.join(self.path, reponame, "last")
+ try:
return int(open(last_path, "r").read().rstrip())
- return 0
+ except Exception:
+ return 0
def get_image(self, reponame, imagename, imageversion):
'''Obtain a local path in cache for a remote image in repo'''
arrow("Getting image", 1, self.verbose)
filename = "%s-%s%s" % (imagename, imageversion, Image.image_extension)
- localpath = os.path.join(self.image_path, filename)
+ cachepath = os.path.join(self.repos[reponame]["cache"].config.image, filename)
# return db path if exists
- if os.path.exists(localpath):
+ if os.path.exists(cachepath):
arrow("Found in cache", 2, self.verbose)
- return localpath
- # get remote image
- remotepath = os.path.join(self.repos[reponame].image_path, filename)
- arrow("Copying from repository", 2, self.verbose)
- istools.copy(remotepath, localpath)
- return localpath
+ else:
+ # get remote image
+ remotepath = os.path.join(self.repos[reponame]["orig"].config.image, filename)
+ arrow("Copying from repository", 2, self.verbose)
+ istools.copy(remotepath, cachepath)
+ return cachepath
def find_image(self, name, version):
'''Find an image in repositories'''
@@ -241,7 +256,7 @@ class RepositoryCache(object):
img = None
# search in all repositories
for repo in self.repos:
- tempdb = Database(os.path.join(self.db_path, repo), False)
+ tempdb = Database(self.repos[repo]["cache"].db.path, False)
img = tempdb.find(name, version)
if img is not None:
# \o/
diff --git a/installsystems/tools.py b/installsystems/tools.py
index b4a084b..1f9340b 100644
--- a/installsystems/tools.py
+++ b/installsystems/tools.py
@@ -9,6 +9,7 @@ InstallSystems Generic Tools Library
import os
import hashlib
import shutil
+import urllib2
def md5sum(path):
'''Compute md5 of a file'''
@@ -16,20 +17,26 @@ def md5sum(path):
m.update(open(path, "r").read())
return m.hexdigest()
-def copy(source, destination, uid=None, gid=None, mode=None):
+def copy(source, destination, uid=None, gid=None, mode=None, timeout=None):
'''Copy a source to destination. Take care of path type'''
stype = pathtype(source)
dtype = pathtype(destination)
+ # ensure destination is not a directory
+ if dtype == "file" and os.path.isdir(destination):
+ destination = os.path.join(destination, os.path.basename(source))
+ # trivial case
if stype == dtype == "file":
shutil.copy(source, destination)
+ elif stype == "http" and dtype == "file":
+ f_dest = open(destination, "w")
+ f_source = urllib2.urlopen(source, timeout=timeout)
+ f_dest.write(f_source.read())
elif stype == "file" and dtype == "":
raise NotImplementedError
else:
raise NotImplementedError
# setting destination file rights
if dtype == "file":
- if os.path.isdir(destination):
- destination = os.path.join(destination, os.path.basename(source))
chrights(destination, uid, gid, mode)
def mkdir(path, uid=None, gid=None, mode=None):
--
GitLab