Introduce Payload

Concept of data was changed to payload.
Payload is not necessary a tarball of a file or a directory, but, directly a file without tarball around.
This is useful have the same md5 on data/payload and win some space during repo adding.
Now data which come with image are represented by class Payload and all operation are located inside. This is cleaner!
Payload also handle orginial information (mtime,uid,gid,md5,size,mode)

installsystems/config.py

@@ -51,7 +51,6 @@ class ConfigFile(object):
                     # get all options in repo
                     self._repos.append(RepositoryConfig(rep, **dict(cp.items(rep))))
             except Exception as e:
-                raise
                 raise Exception("Unable load file %s: %s" % (self.path, e))
         else:
             debug("No config file found")

installsystems/database.py

@@ -52,7 +52,7 @@ class Database(object):
         self.conn.execute("PRAGMA foreign_keys = ON")
 
     def get(self, name, version):
-        '''Return a description dict from a package name'''
+        '''Return a description dict from a image name'''
         # parse tarball
         try:
             self.file.seek(0)
@@ -60,20 +60,20 @@ class Database(object):
             rdata = tarball.get_str("%s-%s" % (name, version))
             tarball.close()
         except KeyError:
-            raise Exception("No package %s version %s in metadata" % (name, version))
+            raise Exception("No image %s version %s in metadata" % (name, version))
         except Exception as e:
             raise Exception("Unable to read db %s version %s: %s" % (name, version, e))
         # convert loaded data into dict (json parser)
         try:
             return json.loads(rdata)
         except Exception as e:
-            raise Exception("Invalid metadata in package %s version %s: e" % (name, version, e))
+            raise Exception("Invalid metadata in image %s version %s: e" % (name, version, e))
 
     def ask(self, sql, args=()):
         '''Ask question to db'''
         return self.conn.execute(sql, args)
 
-    def add(self, package):
+    def add(self, image):
         '''Add a packaged image to a db'''
         try:
             # let's go
@@ -82,28 +82,28 @@ class Database(object):
             # insert image information
             arrow("Add image metadata", 2, self.verbose)
             self.conn.execute("INSERT OR REPLACE INTO image values (?,?,?,?,?,?,?)",
-                              (package.md5,
-                               package.name,
-                               package.version,
-                               package.date,
-                               package.author,
-                               package.description,
-                               package.size,
+                              (image.md5,
+                               image.name,
+                               image.version,
+                               image.date,
+                               image.author,
+                               image.description,
+                               image.size,
                                ))
             # insert data informations
-            arrow("Add data metadata", 2, self.verbose)
-            for key,value in package.data.items():
-                self.conn.execute("INSERT OR REPLACE INTO data values (?,?,?,?)",
-                                  (value["md5"],
-                                   package.md5,
-                                   key,
-                                   value["size"]
+            arrow("Add payload metadata", 2, self.verbose)
+            for name, obj in image.payload.items():
+                self.conn.execute("INSERT OR REPLACE INTO payload values (?,?,?,?,?)",
+                                  (obj.md5,
+                                   image.md5,
+                                   name,
+                                   obj.isdir,
+                                   obj.size,
                                    ))
             # on commit
             arrow("Commit transaction to db", 1, self.verbose)
             self.conn.execute("COMMIT TRANSACTION")
         except Exception as e:
-            raise
             raise Exception("Adding metadata fail: %s" % e)
 
     def delete(self, name, version):

installsystems/repository.py

@@ -77,30 +77,33 @@ class Repository(object):
             raise Exception("Read last file failed: %s" % e)
         return 0
 
-    def add(self, package):
+    def add(self, image):
         '''Add a packaged image to repository'''
         # check local repository
         if istools.pathtype(self.config.path) != "file":
             raise NotImplementedError("Repository addition must be local")
         # checking data tarballs md5 before copy
-        package.check("Check tarballs before copy")
+        image.check("Check image and payload before copy")
         # adding file to repository
-        arrow("Copying files", 1, self.verbose)
-        for src,value in package.tarballs.items():
-            dest = os.path.join(self.config.path, value["md5"])
-            basesrc = os.path.basename(src)
+        arrow("Copying images and payload", 1, self.verbose)
+        for obj in [ image ] + image.payload.values():
+            dest = os.path.join(self.config.path, obj.md5)
+            basesrc = os.path.basename(obj.path)
             if os.path.exists(dest):
                 arrow("Skipping %s: already exists" % basesrc, 2, self.verbose)
             else:
-                arrow("Adding %s (%s)" % (basesrc, value["md5"]), 2, self.verbose)
-                istools.copy(src, dest, self.config.uid, self.config.gid, self.config.fmod)
-        # copy is done. create a package inside repo
-        r_package = PackageImage(os.path.join(self.config.path, package.md5),
+                arrow("Adding %s (%s)" % (basesrc, obj.md5), 2, self.verbose)
+                istools.copy(obj.path, dest,
+                             self.config.uid, self.config.gid, self.config.fmod)
+        # copy is done. create a image inside repo
+        r_image = PackageImage(os.path.join(self.config.path, image.md5),
                                  md5name=True, verbose=self.verbose)
-        # checking data tarballs md5 after copy
-        r_package.check("Check tarballs after copy")
+        # checking must be done with original md5
+        r_image.md5 = image.md5
+        # checking image and payload after copy
+        r_image.check("Check image and payload after copy")
         # add description to db
-        self.db.add(r_package)
+        self.db.add(r_image)
         # update last file
         self.update_last()
 
@@ -137,7 +140,7 @@ class Repository(object):
         return self.db.ask("select name,version from image where name = ? and version = ? limit 1", (name,version)).fetchone() is not None
 
     def get(self, name, version):
-        '''return a package from a name and version of pakage'''
+        '''return a image from a name and version of pakage'''
         # get file md5 from db
         r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
                         (name,version)).fetchone()

installsystems/tarball.py

@@ -36,3 +36,10 @@ class Tarball(tarfile.TarFile):
         else:
             return [ tpname for tpname in lorig
                      if re.match(reg_pattern, tpname) ]
+
+    def size(self):
+        '''Return real (uncompressed) size of the tarball'''
+        total_sz = 0
+        for ti in self.getmembers():
+            total_sz += ti.size
+        return total_sz

installsystems/template.py

@@ -24,7 +24,7 @@ setup = """# -*- python -*-
 
 print "hostname: %s" % args.hostname
 
-image.extractdata("rootfs", args.target)
+image.payload["rootfs"].extract(args.target)
 
 # vim:set ts=2 sw=2 noet:
 """
@@ -39,9 +39,10 @@ CREATE TABLE image (md5 TEXT NOT NULL PRIMARY KEY,
                     size INTEGER NOT NULL,
                     UNIQUE(name, version));
 
-CREATE TABLE data (md5 TEXT NOT NULL,
-                   image_md5 TEXT NOT NULL REFERENCES image(md5),
-                   name TEXT NOT NULL,
-                   size INTEGER NOT NULL,
-                   PRIMARY KEY(md5, image_md5));
+CREATE TABLE payload (md5 TEXT NOT NULL,
+                     image_md5 TEXT NOT NULL REFERENCES image(md5),
+                     name TEXT NOT NULL,
+                     isdir INTEGER NOT NULL,
+                     size INTEGER NOT NULL,
+                     PRIMARY KEY(md5, image_md5));
 """

installsystems/tools.py

@@ -37,7 +37,8 @@ def copyfileobj(sfile, dfile):
             break
         f_len += buf_len
         f_sum.update(buf)
-        dfile.write(buf)
+        if dfile is not None:
+            dfile.write(buf)
     return (f_len , f_sum.hexdigest())
 
 def copy(source, destination, uid=None, gid=None, mode=None, timeout=None):
@@ -67,7 +68,7 @@ def mkdir(path, uid=None, gid=None, mode=None):
     os.mkdir(path)
     chrights(path, uid, gid, mode)
 
-def chrights(path, uid=None, gid=None, mode=None):
+def chrights(path, uid=None, gid=None, mode=None, mtime=None):
     '''Set rights on a file'''
     if uid is not None:
         os.chown(path, uid, -1)
@@ -75,6 +76,8 @@ def chrights(path, uid=None, gid=None, mode=None):
         os.chown(path, -1, gid)
     if mode is not None:
         os.chmod(path, mode)
+    if mtime is not None:
+        os.utime(path, (mtime, mtime))
 
 def pathtype(path):
     '''Return path type. This is usefull to know what kind of path is given'''
@@ -103,14 +106,26 @@ def abspath(path):
     else:
         return None
 
-def uopen(path):
+def uopen(path, mode="rb"):
     '''Universal Open
     Create a file-like object to a file which can be remote
     '''
     ftype = pathtype(path)
     if ftype == "file":
-        return open(path, "r")
+        return open(path, mode)
     elif ftype == "http" or ftype == "ftp":
         return urllib2.urlopen(path)
     else:
         raise NotImplementedError
+
+def getsize(path):
+    '''Get size of a path. Recurse if directory'''
+    total_sz = os.path.getsize(path)
+    if os.path.isdir(path):
+        for root, dirs, files in os.walk(path):
+            for filename in dirs + files:
+                filepath = os.path.join(root, filename)
+                filestat = os.lstat(filepath)
+                if stat.S_ISDIR(filestat.st_mode) or stat.S_ISREG(filestat.st_mode):
+                    total_sz += filestat.st_size
+    return total_sz