Commit bf679786 authored by Seblu's avatar Seblu
Browse files

Introduce Payload 

Concept of data was changed to payload.
Payload is not necessary a tarball of a file or a directory, but, directly a file without tarball around.
This is useful have the same md5 on data/payload and win some space during repo adding.
Now data which come with image are represented by class Payload and all operation are located inside. This is cleaner!
Payload also handle orginial information (mtime,uid,gid,md5,size,mode)
parent f922bcd1

installsystems/config.py

+0 −1
Original line number Diff line number Diff line
@@ -51,7 +51,6 @@ class ConfigFile(object): 
                    # get all options in repo

                    self._repos.append(RepositoryConfig(rep, **dict(cp.items(rep))))

            except Exception as e:

                raise

                raise Exception("Unable load file %s: %s" % (self.path, e))

        else:

            debug("No config file found")

installsystems/database.py

+19 −19
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ class Database(object): 
        self.conn.execute("PRAGMA foreign_keys = ON")



    def get(self, name, version):

        '''Return a description dict from a package name'''

        '''Return a description dict from a image name'''

        # parse tarball

        try:

            self.file.seek(0)
@@ -60,20 +60,20 @@ class Database(object): 
            rdata = tarball.get_str("%s-%s" % (name, version))

            tarball.close()

        except KeyError:

            raise Exception("No package %s version %s in metadata" % (name, version))

            raise Exception("No image %s version %s in metadata" % (name, version))

        except Exception as e:

            raise Exception("Unable to read db %s version %s: %s" % (name, version, e))

        # convert loaded data into dict (json parser)

        try:

            return json.loads(rdata)

        except Exception as e:

            raise Exception("Invalid metadata in package %s version %s: e" % (name, version, e))

            raise Exception("Invalid metadata in image %s version %s: e" % (name, version, e))



    def ask(self, sql, args=()):

        '''Ask question to db'''

        return self.conn.execute(sql, args)



    def add(self, package):

    def add(self, image):

        '''Add a packaged image to a db'''

        try:

            # let's go
@@ -82,28 +82,28 @@ class Database(object): 
            # insert image information

            arrow("Add image metadata", 2, self.verbose)

            self.conn.execute("INSERT OR REPLACE INTO image values (?,?,?,?,?,?,?)",

                              (package.md5,

                               package.name,

                               package.version,

                               package.date,

                               package.author,

                               package.description,

                               package.size,

                              (image.md5,

                               image.name,

                               image.version,

                               image.date,

                               image.author,

                               image.description,

                               image.size,

                               ))

            # insert data informations

            arrow("Add data metadata", 2, self.verbose)

            for key,value in package.data.items():

                self.conn.execute("INSERT OR REPLACE INTO data values (?,?,?,?)",

                                  (value["md5"],

                                   package.md5,

                                   key,

                                   value["size"]

            arrow("Add payload metadata", 2, self.verbose)

            for name, obj in image.payload.items():

                self.conn.execute("INSERT OR REPLACE INTO payload values (?,?,?,?,?)",

                                  (obj.md5,

                                   image.md5,

                                   name,

                                   obj.isdir,

                                   obj.size,

                                   ))

            # on commit

            arrow("Commit transaction to db", 1, self.verbose)

            self.conn.execute("COMMIT TRANSACTION")

        except Exception as e:

            raise

            raise Exception("Adding metadata fail: %s" % e)



    def delete(self, name, version):

installsystems/image.py

+366 −141

File changed.

Preview size limit exceeded, changes collapsed.

installsystems/repository.py

+17 −14
Original line number Diff line number Diff line
@@ -77,30 +77,33 @@ class Repository(object): 
            raise Exception("Read last file failed: %s" % e)

        return 0



    def add(self, package):

    def add(self, image):

        '''Add a packaged image to repository'''

        # check local repository

        if istools.pathtype(self.config.path) != "file":

            raise NotImplementedError("Repository addition must be local")

        # checking data tarballs md5 before copy

        package.check("Check tarballs before copy")

        image.check("Check image and payload before copy")

        # adding file to repository

        arrow("Copying files", 1, self.verbose)

        for src,value in package.tarballs.items():

            dest = os.path.join(self.config.path, value["md5"])

            basesrc = os.path.basename(src)

        arrow("Copying images and payload", 1, self.verbose)

        for obj in [ image ] + image.payload.values():

            dest = os.path.join(self.config.path, obj.md5)

            basesrc = os.path.basename(obj.path)

            if os.path.exists(dest):

                arrow("Skipping %s: already exists" % basesrc, 2, self.verbose)

            else:

                arrow("Adding %s (%s)" % (basesrc, value["md5"]), 2, self.verbose)

                istools.copy(src, dest, self.config.uid, self.config.gid, self.config.fmod)

        # copy is done. create a package inside repo

        r_package = PackageImage(os.path.join(self.config.path, package.md5),

                arrow("Adding %s (%s)" % (basesrc, obj.md5), 2, self.verbose)

                istools.copy(obj.path, dest,

                             self.config.uid, self.config.gid, self.config.fmod)

        # copy is done. create a image inside repo

        r_image = PackageImage(os.path.join(self.config.path, image.md5),

                                 md5name=True, verbose=self.verbose)

        # checking data tarballs md5 after copy

        r_package.check("Check tarballs after copy")

        # checking must be done with original md5

        r_image.md5 = image.md5

        # checking image and payload after copy

        r_image.check("Check image and payload after copy")

        # add description to db

        self.db.add(r_package)

        self.db.add(r_image)

        # update last file

        self.update_last()
@@ -137,7 +140,7 @@ class Repository(object): 
        return self.db.ask("select name,version from image where name = ? and version = ? limit 1", (name,version)).fetchone() is not None



    def get(self, name, version):

        '''return a package from a name and version of pakage'''

        '''return a image from a name and version of pakage'''

        # get file md5 from db

        r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",

                        (name,version)).fetchone()

installsystems/tarball.py

+7 −0
Original line number Diff line number Diff line
@@ -36,3 +36,10 @@ class Tarball(tarfile.TarFile): 
        else:

            return [ tpname for tpname in lorig

                     if re.match(reg_pattern, tpname) ]



    def size(self):

        '''Return real (uncompressed) size of the tarball'''

        total_sz = 0

        for ti in self.getmembers():

            total_sz += ti.size

        return total_sz