Improve remote cache. Http transport enabled

bin/isinstall

@@ -45,9 +45,9 @@ try:
     # load config
     config = ConfigFile("isinstall", args.config)
     # looks if arguments is a file or image name
-    image_name_type = istools.get_path_type(args.image_name)
+    image_name_type = istools.pathtype(args.image_name)
     if image_name_type == "file":
-        pkg = PackageImage(istools.complete_path(args.image_name))
+        pkg = PackageImage(istools.abspath(args.image_name))
     elif image_name_type == "name":
         # init repo cache object
         repocache = RepositoryCache(config.cache, verbose=args.verbose)

installsystems/config.py

@@ -32,7 +32,7 @@ class ConfigFile(object):
         '''Load/Reload config file'''
         # seting default config
         self._config = {}
-        self._repos = {}
+        self._repos = []
         # loading config file if exists
         if self.path is not None:
             debug("Loading config file: %s" % self.path)
@@ -49,7 +49,7 @@ class ConfigFile(object):
                     if "image" not in cp.options(rep):
                         continue
                     # get all options in repo
-                    self._repos[rep] = RepositoryConfig(rep, **dict(cp.items(rep)))
+                    self._repos.append( RepositoryConfig(rep, **dict(cp.items(rep))))
             except Exception as e:
                 raise
                 raise Exception("Unable load file %s: %s" % (self.path, e))
@@ -89,5 +89,6 @@ class ConfigFile(object):
 
     @property
     def repos(self):
-        '''Get a list of repository available'''
-        return self._repos.copy()
+        '''Get a dict of repository available'''
+        # deep copy
+        return list(self._repos)

installsystems/database.py

@@ -22,7 +22,7 @@ class Database(object):
     @classmethod
     def create(cls, path, verbose=True):
         arrow("Creating repository database", 1, verbose)
-        dbpath = os.path.abspath(path)
+        dbpath = istools.abspath(path)
         if os.path.exists(dbpath):
             raise Exception("db already exists")
         try:
@@ -34,7 +34,7 @@ class Database(object):
         return cls(path, verbose)
 
     def __init__(self, path, verbose=True):
-        self.path = os.path.abspath(path)
+        self.path = istools.abspath(path)
         self.verbose = verbose
 
     def add(self, package):

installsystems/repository.py

@@ -11,6 +11,7 @@ import time
 import shutil
 import pwd
 import grp
+import copy
 import installsystems
 import installsystems.tools as istools
 from installsystems.printer import *
@@ -176,61 +177,75 @@ class RepositoryCache(object):
     '''Local repository cache class'''
 
     def __init__(self, cache_path, verbose=True):
-        self.base_path = os.path.abspath(cache_path)
-        self.image_path = os.path.join(self.base_path, "image")
-        self.last_path = os.path.join(self.base_path, "last")
-        self.db_path = os.path.join(self.base_path, "db")
-        for path in (self.base_path, self.image_path, self.last_path, self.db_path):
-            if not os.path.exists(path):
-                os.mkdir(path)
-            if not os.access(path, os.W_OK | os.X_OK):
-                raise Exception("%s is not writable or executable" % path)
         self.verbose = verbose
-        self.repos = dict()
-        debug("Repository cache is in %s" % self.base_path)
+        self.repos = {}
+        self.path = os.path.abspath(cache_path)
+        # ensure cache directories are avaiblable
+        if not os.path.exists(self.path):
+            os.mkdir(self.path)
+        if not os.access(self.path, os.W_OK | os.X_OK):
+            raise Exception("%s is not writable or executable" % path)
+        debug("Repository cache is in %s" % self.path)
 
-    def register(self, iterepo):
-        '''Register a repository to track'''
-        for r in iterepo:
-            self.repos[r[0]] = Repository(istools.abspath(r[1]),
-                                          istools.abstpath(r[2]),
-                                          verbose=self.verbose)
+    def register(self, configs):
+        '''Register a list of repository from its config'''
+        for conf in configs:
+            self.repos[conf.name] = {}
+            # keep original repository conf
+            self.repos[conf.name]["orig"] = Repository(conf, self.verbose)
+            # change configuration to make remote repository in cache
+            cconf = copy.copy(conf)
+            cconf.image = os.path.join(self.path, conf.name)
+            cconf.data = "/dev/null"
+            self.repos[conf.name]["cache"] = Repository(cconf, self.verbose)
+            # create a local directory
+            if not os.path.exists(cconf.image):
+                os.mkdir(cconf.image)
 
     def update(self):
         '''Update cache info'''
         arrow("Updating repositories", 1, self.verbose)
         for r in self.repos:
-            debug("%s: remote_last: %s, local_last:%s" % (r,
-                                                          self.repos[r].last(),
-                                                          self.last(r)))
-            if self.repos[r].last() > self.last(r):
-                # copy last file
-                istools.copy(self.repos[r].last_path, os.path.join(self.last_path, r))
+            # last local
+            local_last = self.last(r)
+            # copy last file
+            arrow("Copying %s repository last" % r, 2, self.verbose)
+            istools.copy(self.repos[r]["orig"].last_path,
+                         self.repos[r]["cache"].last_path,)
+            # last after update
+            remote_last = self.last(r)
+            debug("%s: last: local: %s, remote:%s" % (r, local_last, remote_last))
+            # Updating db?
+            remote_db = self.repos[r]["orig"].db.path
+            local_db = self.repos[r]["cache"].db.path
+            if remote_last > local_last or not os.path.exists(local_db):
                 # copy db file
-                istools.copy(self.repos[r].db.path, os.path.join(self.db_path, r))
+                arrow("Copying %s repository db" % r, 2, self.verbose)
+                istools.copy(remote_db, local_db)
                 arrow("%s updated" % r, 2, self.verbose)
 
     def last(self, reponame):
         '''Return the last timestamp of a repo'''
-        last_path = os.path.join(self.last_path, reponame)
-        if os.path.exists(last_path):
+        last_path = os.path.join(self.path, reponame, "last")
+        try:
             return int(open(last_path, "r").read().rstrip())
-        return 0
+        except Exception:
+            return 0
 
     def get_image(self, reponame, imagename, imageversion):
         '''Obtain a local path in cache for a remote image in repo'''
         arrow("Getting image", 1, self.verbose)
         filename = "%s-%s%s" % (imagename, imageversion, Image.image_extension)
-        localpath = os.path.join(self.image_path, filename)
+        cachepath = os.path.join(self.repos[reponame]["cache"].config.image, filename)
         # return db path if exists
-        if os.path.exists(localpath):
+        if os.path.exists(cachepath):
             arrow("Found in cache", 2, self.verbose)
-            return localpath
-        # get remote image
-        remotepath = os.path.join(self.repos[reponame].image_path, filename)
-        arrow("Copying from repository", 2, self.verbose)
-        istools.copy(remotepath, localpath)
-        return localpath
+        else:
+            # get remote image
+            remotepath = os.path.join(self.repos[reponame]["orig"].config.image, filename)
+            arrow("Copying from repository", 2, self.verbose)
+            istools.copy(remotepath, cachepath)
+        return cachepath
 
     def find_image(self, name, version):
         '''Find an image in repositories'''
@@ -241,7 +256,7 @@ class RepositoryCache(object):
         img = None
         # search in all repositories
         for repo in self.repos:
-            tempdb = Database(os.path.join(self.db_path, repo), False)
+            tempdb = Database(self.repos[repo]["cache"].db.path, False)
             img = tempdb.find(name, version)
             if img is not None:
                 # \o/

installsystems/tools.py

@@ -9,6 +9,7 @@ InstallSystems Generic Tools Library
 import os
 import hashlib
 import shutil
+import urllib2
 
 def md5sum(path):
     '''Compute md5 of a file'''
@@ -16,20 +17,26 @@ def md5sum(path):
     m.update(open(path, "r").read())
     return m.hexdigest()
 
-def copy(source, destination, uid=None, gid=None, mode=None):
+def copy(source, destination, uid=None, gid=None, mode=None, timeout=None):
     '''Copy a source to destination. Take care of path type'''
     stype = pathtype(source)
     dtype = pathtype(destination)
+    # ensure destination is not a directory
+    if dtype == "file" and os.path.isdir(destination):
+        destination = os.path.join(destination, os.path.basename(source))
+    # trivial case
     if stype == dtype == "file":
         shutil.copy(source, destination)
+    elif stype == "http" and dtype == "file":
+        f_dest = open(destination, "w")
+        f_source = urllib2.urlopen(source, timeout=timeout)
+        f_dest.write(f_source.read())
     elif stype == "file" and dtype == "":
         raise NotImplementedError
     else:
         raise NotImplementedError
     # setting destination file rights
     if dtype == "file":
-        if os.path.isdir(destination):
-            destination = os.path.join(destination, os.path.basename(source))
         chrights(destination, uid, gid, mode)
 
 def mkdir(path, uid=None, gid=None, mode=None):