image.py

            for payname in self.payload:
                arrow(u"Downloading payload %s in %s" % (payname, directory))
                self.payload[payname].info
                self.payload[payname].download(directory, force=force)

    def extract(self, directory, force=False, payload=False, gendescription=False):
        '''
        Extract content of the image inside a repository
        '''
        # check validity of dest
        if os.path.exists(directory):
            if not os.path.isdir(directory):
                raise ISError(u"Destination %s is not a directory" % directory)
            if not force and len(os.listdir(directory)) > 0:
                raise ISError(u"Directory %s is not empty (need force)" % directory)
        else:
            istools.mkdir(directory)
        # extract content
        arrow(u"Extracting image in %s" % directory)
        self._tarball.extractall(directory)
        # generate description file from description.json
        if gendescription:
            arrow(u"Generating description file in %s" % directory)
            with open(os.path.join(directory, "description"), "w") as f:
                f.write((istemplate.description % self._metadata).encode("UTF-8"))
        # launch payload extraction
        if payload:
            for payname in self.payload:
                # here we need to decode payname which is in unicode to escape
                # tarfile to encode filename of file inside tarball inside unicode
                dest = os.path.join(directory, "payload", payname.encode("UTF-8"))
                arrow(u"Extracting payload %s in %s" % (payname, dest))
                self.payload[payname].extract(dest, force=force)

    def run(self, parser, extparser, load_modules=True, run_parser=True,
            run_setup=True):
        '''
        Run images scripts

        parser is the whole command line parser
        extparser is the parser extensible by parser scripts

        if load_modules is true load image modules
        if run_parser is true run parser scripts
        if run_setup is true run setup scripts
        '''
        # register start time
        t0 = time.time()
        # load image modules
        if load_modules:
            self.load_modules(lambda: self.select_scripts("lib"))
        # run parser scripts to extend extparser
        # those scripts should only extend the parser or produce error
        if run_parser:
            self.run_scripts("parser",
                             lambda: self.select_scripts("parser"),
                             "/",
                             {"parser": extparser})
        # call parser (again), with full options
        arrow("Parsing command line")
        # encode command line arguments to utf-8
        args = istools.argv()[1:]
        # catch exception in custom argparse action
        try:
            args = parser.parse_args(args=args)
        except Exception as e:
            raise ISError("Argument parser", e)
        # run setup scripts
        if run_setup:
            self.run_scripts("setup",
                             lambda: self.select_scripts("setup"),
                             "/",
                             {"namespace": args})
        # return the building time
        return int(time.time() - t0)

    def select_scripts(self, directory):
        '''
        Generator of tuples (fp,fn,fc) of scripts witch are allocatable
        in a tarball directory
        '''
        for fp in sorted(self._tarball.getnames(re_pattern="%s/.*\.py" % directory)):
            fn = os.path.basename(fp)
            # extract source code
            try:
                fc = self._tarball.get_str(fp)
            except Exception as e:
                raise ISError(u"Unable to extract script %s" % fp, e)
            # yield complet file path, file name and file content
            yield (fp, fn, fc)


class Payload(object):
    '''
    Payload class represents a payload object
    '''
    extension = ".isdata"
    legit_attr = ("isdir", "md5", "size", "uid", "gid", "mode", "mtime", "compressor")

    def __init__(self, name, filename, path, **kwargs):
        object.__setattr__(self, "name", name)
        object.__setattr__(self, "filename", filename)
        object.__setattr__(self, "path", path)
        # register legit param
        for attr in self.legit_attr:
            setattr(self, attr, None)
        # set all named param
        for kwarg in kwargs:
            # do not use hasattr which use getattr and so call md5 checksum...
            if kwarg in self.legit_attr:
                setattr(self, kwarg, kwargs[kwarg])

    def __getattr__(self, name):
        # get all value with an understance as if there is no underscore
        if hasattr(self, u"_%s" % name):
            return getattr(self, u"_%s" % name)
        raise AttributeError

    def __setattr__(self, name, value):
        # set all value which exists have no underscore, but where underscore exists
        if name in self.legit_attr:
            object.__setattr__(self, u"_%s" % name, value)
        else:
            object.__setattr__(self, name, value)

    def checksummize(self):
        '''
        Fill missing md5/size about payload
        '''
        fileobj = PipeFile(self.path, "r")
        fileobj.consume()
        fileobj.close()
        if self._size is None:
            self._size = fileobj.read_size
        if self._md5 is None:
            self._md5 = fileobj.md5

    @property
    def md5(self):
        '''
        Return md5 of payload
        '''
        if self._md5 is None:
            self.checksummize()
        return self._md5

    @property
    def size(self):
        '''
        Return size of payload
        '''
        if self._size is None:
            self.checksummize()
        return self._size

    @property
    def uid(self):
        '''
        Return uid of owner of orginal payload
        '''
        return self._uid if self._uid is not None else 0

    @property
    def gid(self):
        '''
        Return gid of owner of orginal payload
        '''
        return self._gid if self._gid is not None else 0

    @property
    def mode(self):
        '''
        Return mode of orginal payload
        '''
        if self._mode is not None:
            return self._mode
        else:
            umask = os.umask(0)
            os.umask(umask)
            return 0666 & ~umask

    @property
    def mtime(self):
        '''
        Return last modification time of orginal payload
        '''
        return self._mtime if self._mtime is not None else time.time()

    @property
    def compressor(self):
        '''
        Return payload compress format
        '''
        return self._compressor if self._compressor is not None else Image.default_compressor

    @property
    def info(self):
        '''
        Return a dict of info about current payload
        Auto calculated info like name and filename must not be here
        '''
        return {"md5": self.md5,
                "size": self.size,
                "isdir": self.isdir,
                "uid": self.uid,
                "gid": self.gid,
                "mode": self.mode,
                "mtime": self.mtime}

    def check(self):
        '''
        Check that path correspond to current md5 and size
        '''
        if self._size is None or self._md5 is None:
            debug("Check is called on payload with nothing to check")
            return True
        fileobj = PipeFile(self.path, "r")
        fileobj.consume()
        fileobj.close()
        if self._size != fileobj.read_size:
            raise ISError(u"Invalid size of payload %s" % self.name)
        if self._md5 != fileobj.md5:
            raise ISError(u"Invalid MD5 of payload %s" % self._md5)

    def download(self, dest, force=False):
        '''
        Download payload in directory
        '''
        # if dest is a directory try to create file inside
        if os.path.isdir(dest):
            dest = os.path.join(dest, self.filename)
        # try to create leading directories
        elif not os.path.exists(os.path.dirname(dest)):
            istools.mkdir(os.path.dirname(dest))
        # check validity of dest
        if os.path.exists(dest):
            if os.path.isdir(dest):
                raise ISError(u"Destination %s is a directory" % dest)
            if not force:
                raise ISError(u"File %s already exists" % dest)
        # open remote file
        debug(u"Downloading payload %s from %s" % (self.filename, self.path))
        fs = PipeFile(self.path, progressbar=True)
        # check if announced file size is good
        if fs.size is not None and self.size != fs.size:
            raise ISError(u"Downloading payload %s failed: Invalid announced size" %
                            self.name)
        fd = open(dest, "wb")
        fs.consume(fd)
        # closing fo
        fs.close()
        fd.close()
        # checking download size
        if self.size != fs.read_size:
            raise ISError(u"Downloading payload %s failed: Invalid size" % self.name)
        if self.md5 != fs.md5:
            raise ISError(u"Downloading payload %s failed: Invalid MD5" % self.name)

    def extract(self, dest, force=False, filelist=None):
        '''
        Extract payload into dest
        filelist is a filter of file in tarball
        force will overwrite existing file if exists
        '''
        try:
            if self.isdir:
                self.extract_tar(dest, force=force, filelist=filelist)
            else:
                self.extract_file(dest, force=force)
        except Exception as e:
            raise ISError(u"Extracting payload %s failed" % self.name, e)

    def extract_tar(self, dest, force=False, filelist=None):
        '''
        Extract a payload which is a tarball.
        This is used mainly to extract payload from a directory
        '''
        # check validity of dest
        if os.path.exists(dest):
            if not os.path.isdir(dest):
                raise ISError(u"Destination %s is not a directory" % dest)
            if not force and len(os.listdir(dest)) > 0:
                raise ISError(u"Directory %s is not empty (need force)" % dest)
        else:
            istools.mkdir(dest)
        # try to open payload file
        try:
            fo = PipeFile(self.path, progressbar=True)
        except Exception as e:
            raise ISError(u"Unable to open %s" % self.path)
        # check if announced file size is good
        if fo.size is not None and self.size != fo.size:
            raise ISError(u"Invalid announced size on %s" % self.path)
        # get compressor argv (first to escape file creation if not found)
        a_comp = istools.get_compressor_path(self.compressor, compress=False)
        a_tar = ["tar", "--extract", "--numeric-owner", "--ignore-zeros",
                 "--preserve-permissions", "--directory", dest]
        # add optionnal selected filename for decompression
        if filelist is not None:
            a_tar += filelist
        p_tar = subprocess.Popen(a_tar, shell=False, close_fds=True,
                                 stdin=subprocess.PIPE)
        p_comp = subprocess.Popen(a_comp, shell=False, close_fds=True,
                                  stdin=subprocess.PIPE, stdout=p_tar.stdin)
        # close tar fd
        p_tar.stdin.close()
        # push data into compressor
        fo.consume(p_comp.stdin)
        # close source fd
        fo.close()
        # checking downloaded size
        if self.size != fo.read_size:
            raise ISError("Invalid size")
        # checking downloaded md5
        if self.md5 != fo.md5:
            raise ISError("Invalid MD5")
        # close compressor pipe
        p_comp.stdin.close()
        # check compressor return 0
        if p_comp.wait() != 0:
            raise ISError(u"Compressor %s return is not zero" % a_comp[0])
        # check tar return 0
        if p_tar.wait() != 0:
            raise ISError("Tar return is not zero")

    def extract_file(self, dest, force=False):
        '''
        Copy a payload directly to a file
        Check md5 on the fly
        '''
        # if dest is a directory try to create file inside
        if os.path.isdir(dest):
            dest = os.path.join(dest, self.name)
        # try to create leading directories
        elif not os.path.exists(os.path.dirname(dest)):
            istools.mkdir(os.path.dirname(dest))
        # check validity of dest
        if os.path.exists(dest):
            if os.path.isdir(dest):
                raise ISError(u"Destination %s is a directory" % dest)
            if not force:
                raise ISError(u"File %s already exists" % dest)
        # get compressor argv (first to escape file creation if not found)
        a_comp = istools.get_compressor_path(self.compressor, compress=False)
        # try to open payload file (source)
        try:
            f_src = PipeFile(self.path, "r", progressbar=True)
        except Exception as e:
            raise ISError(u"Unable to open payload file %s" % self.path, e)
        # check if announced file size is good
        if f_src.size is not None and self.size != f_src.size:
            raise ISError(u"Invalid announced size on %s" % self.path)
        # opening destination
        try:
            f_dst = open(dest, "wb")
        except Exception as e:
            raise ISError(u"Unable to open destination file %s" % dest, e)
        # run compressor process
        p_comp = subprocess.Popen(a_comp, shell=False, close_fds=True,
                                  stdin=subprocess.PIPE, stdout=f_dst)
        # close destination file
        f_dst.close()
        # push data into compressor
        f_src.consume(p_comp.stdin)
        # closing source fo
        f_src.close()
        # checking download size
        if self.size != f_src.read_size:
            raise ISError("Invalid size")
        # checking downloaded md5
        if self.md5 != f_src.md5:
            raise ISError("Invalid MD5")
        # close compressor pipe
        p_comp.stdin.close()
        # check compressor return 0
        if p_comp.wait() != 0:
            raise ISError(u"Compressor %s return is not zero" % a_comp[0])
        # settings file orginal rights
        istools.chrights(dest, self.uid, self.gid, self.mode, self.mtime)


class Changelog(dict):
    '''
    Object representing a changelog in memory
    '''
    def __init__(self, data):
        self.verbatim = u""
        self.load(data)

    def load(self, data):
        '''
        Load a changelog file
        '''
        # ensure data are correct UTF-8
        if isinstance(data, str):
            try:
                data = unicode(data, "UTF-8")
            except UnicodeDecodeError:
                raise ISError("Invalid character encoding in changelog")
        version = None
        lines = data.split("\n")
        for line in lines:
            # ignore empty lines
            if len(line.strip()) == 0:
                continue
            # ignore comments
            if line.lstrip().startswith("#"):
                continue
            # try to match a new version
            m = re.match("\[(\d+(?:\.\d+)*)(?:([~+]).*)?\]", line.lstrip())
            if m is not None:
                version = m.group(1)
                self[version] = []
                continue
            # if line are out of a version => invalid format
            if version is None:
                raise ISError("Invalid format: Line outside version")
            # add line to version changelog
            self[version] += [line]
        # save original
        self.verbatim = data

    def show(self, version=None, verbose=False):
        '''
        Show changelog for a given version or all
        '''
        out('#light##yellow#Changelog:#reset#')
        # if no version take the hightest
        if version is None:
            version = max(self)
        # display asked version
        if version in self:
            self._show_version(version)
        # display all version in verbose mode
        if verbose:
            for ver in sorted((k for k in self if k < version), reverse=True):
                self._show_version(ver)

    def _show_version(self, version):
        '''
        Display a version content
        '''
        out(u'  #yellow#Version:#reset# %s' % version)
        out(os.linesep.join(self[version]))