Newer
Older
# -*- coding: utf-8 -*-
# Started 26/05/2011 by Seblu <seblu@seblu.net>
'''
InstallSystems Generic Tools Library
'''
import os
from subprocess import call, check_call, CalledProcessError
from installsystems.progressbar import ProgressBar, Percentage
from installsystems.progressbar import FileTransferSpeed, FileTransferSize
from installsystems.progressbar import Bar, BouncingBar, ETA, UnknownLength
from installsystems.tarball import Tarball
from installsystems.printer import *
################################################################################
# Classes
################################################################################
class PipeFile(object):
Pipe file object if a file object with extended capabilties
like printing progress bar or compute file size, md5 on the fly
def __init__(self, path=None, mode="r", fileobj=None, timeout=3,
progressbar=False):
self.open(path, mode, fileobj, timeout)
# start progressbar display if asked
self.progressbar = progressbar
def open(self, path=None, mode="r", fileobj=None, timeout=3):
if path is None and fileobj is None:
raise AttributeError("You must have a path or a fileobj to open")
if mode not in ("r", "w"):
raise AttributeError("Invalid open mode. Must be r or w")
self.mode = mode
if fileobj is not None:
self.fo = fileobj
# seek to 0 and compute filesize if we have and fd
if hasattr(self.fo, "fileno"):
self.seek(0)
self.size = os.fstat(self.fo.fileno()).st_size
else:
ftype = pathtype(path)
if ftype == "file":
self._open_local(path)
elif ftype == "http":
self._open_http(path)
elif ftype == "ftp":
self._open_ftp(path)
elif ftype == "ssh":
self._open_ssh(path)
# we use 0 because a null file is cannot show a progression during write
if self.size == 0:
widget = [ FileTransferSize(), " ", BouncingBar(), " ", FileTransferSpeed() ]
maxval = UnknownLength
else:
widget = [ Percentage(), " ", Bar(), " ", FileTransferSpeed(), " ", ETA() ]
maxval = self.size
self._progressbar = ProgressBar(widgets=widget, maxval=maxval)
def _open_local(self, path):
'''
Open file on the local filesystem
'''
self.fo = open(path, self.mode)
sta = os.fstat(self.fo.fileno())
self.size = sta.st_size
self.mtime = sta.st_mtime
def _open_http(self, path):
'''
Open a file accross an http server
'''
try:
self.fo = urllib2.urlopen(path, timeout=self.timeout)
except Exception as e:
# FIXME: unable to open file
raise IOError(e)
# get file size
if "Content-Length" in self.fo.headers:
self.size = int(self.fo.headers["Content-Length"])
else:
# get mtime
try:
self.mtime = int(time.mktime(time.strptime(self.fo.headers["Last-Modified"],
"%a, %d %b %Y %H:%M:%S %Z")))
except:
self.mtime = None
def _open_ftp(self, path):
'''
Open file via ftp
'''
try:
self.fo = urllib2.urlopen(path, timeout=self.timeout)
except Exception as e:
# FIXME: unable to open file
raise IOError(e)
# get file size
try:
self.size = int(self.fo.headers["content-length"])
except:
def _open_ssh(self, path):
'''
Open current fo from an ssh connection
'''
# try to load paramiko
try:
import paramiko
except ImportError:
raise IOError("URL type not supported")
# parse url
(login, passwd, host, port, path) = re.match(
"ssh://(([^:]+)(:([^@]+))?@)?([^/:]+)(:(\d+))?(/.*)?", path).group(2, 4, 5, 7, 8)
if port is None: port = 22
if path is None: path = "/"
# open ssh connection
# we need to keep it inside the object unless it was cutted
self._ssh = paramiko.SSHClient()
self._ssh.load_system_host_keys()
self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self._ssh.connect(host, port=port, username=login, password=passwd,
look_for_keys=True,
timeout=int(self.timeout))
# swith in sftp mode
sftp = self._ssh.open_sftp()
# get the file infos
sta = sftp.stat(path)
self.size = sta.st_size
self.mtime = sta.st_mtime
# open the file
self.fo = sftp.open(path, self.mode)
# this is needed to have correct file transfert speed
self.fo.set_pipelined(True)
def close(self):
if self.progressbar:
self._progressbar.finish()
debug("MD5: %s" % self.md5)
self.fo.close()
def read(self, size=None):
if self.mode == "w":
raise IOError("Unable to read in w mode")
buf = self.fo.read(size)
length = len(buf)
self._md5.update(buf)
self.consumed_size += length
if self.progressbar and length > 0:
self._progressbar.update(self.consumed_size)
return buf
def flush(self):
if hasattr(self.fo, "flush"):
return self.fo.flush()
def write(self, buf):
if self.mode == "r":
raise IOError("Unable to write in r mode")
length = len(buf)
self._md5.update(buf)
self.consumed_size += length
if self.progressbar and length > 0:
self._progressbar.update(self.consumed_size)
return None
Consume (read) all data and write it in fo
if fo is None, data are discarded. This is useful to obtain md5 and size
Useful to obtain md5 and size
'''
if self.mode == "w":
raise IOError("Unable to read in w mode")
while True:
if fo is not None:
fo.write(buf)
@property
def progressbar(self):
'''
Return is progressbar have been started
'''
return hasattr(self, "_progressbar_started")
@progressbar.setter
def progressbar(self, val):
'''
Set this property to true enable progress bar
'''
if installsystems.quiet is True:
return
if val == True and not hasattr(self, "_progressbar_started"):
self._progressbar_started = True
self._progressbar.start()
@property
def md5(self):
'''
Return the md5 of read/write of the file
'''
return self._md5.hexdigest()
@property
def read_size(self):
'''
Return the current read size
'''
return self.consumed_size
@property
def write_size(self):
'''
Return the current wrote size
'''
return self.consumed_size
################################################################################
# Functions
################################################################################
def smd5sum(buf):
'''
Compute md5 of a string
'''
m = hashlib.md5()
m.update(buf)
return m.hexdigest()
def mkdir(path, uid=None, gid=None, mode=None):
chrights(path, uid, gid, mode)
def chrights(path, uid=None, gid=None, mode=None, mtime=None):
if uid is not None:
os.chown(path, uid, -1)
if gid is not None:
os.chown(path, -1, gid)
if mode is not None:
os.chmod(path, mode)
'''
Return path type. This is usefull to know what kind of path is given
'''
if path.startswith("http://") or path.startswith("https://"):
return "http"
if path.startswith("ftp://") or path.startswith("ftps://"):
return "ftp"
def isfile(path):
'''
Return True if path is of type file
'''
return pathtype(path) == "file"
return path
elif ptype == "file":
if path.startswith("file://"):
return os.path.abspath(path)
else:
return None
total_sz = os.path.getsize(path)
if os.path.isdir(path):
for root, dirs, files in os.walk(path):
for filename in dirs + files:
filepath = os.path.join(root, filename)
filestat = os.lstat(filepath)
if stat.S_ISDIR(filestat.st_mode) or stat.S_ISREG(filestat.st_mode):
total_sz += filestat.st_size
return total_sz
prefixes = ('','Ki', 'Mi', 'Gi', 'Ti','Pi', 'Ei', 'Zi', 'Yi')
power = int(math.log(num, 1024))
# max is YiB
if power >= len(prefixes):
power = len(prefixes) - 1
scaled = num / float(1024 ** power)
return "%3.1f%s%s" % (scaled, prefixes[power], unit)
def time_rfc2822(timestamp):
'''
Return a rfc2822 format time string from an unix timestamp
'''
return time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime(timestamp))
Try to detect which distro is inside a directory
'''
if os.path.exists(os.path.join(path, "etc/debian_version")):
elif os.path.exists(os.path.join(path, "etc/arch-release")):
return "archlinux"
return None
def prepare_chroot(path, mount=True):
'''
Preate a chroot environment by mouting /{proc,sys,dev,dev/pts}
and try to guess dest os to avoid daemon lauching
'''
# try to mount /proc /sys /dev /dev/pts /dev/shm
mps = ("proc", "sys", "dev", "dev/pts", "dev/shm")
arrow("Mouting filesystems")
for mp in mps:
origin = "/%s" % mp
target = os.path.join(path, mp)
if os.path.ismount(target):
warn("%s is already a mountpoint, skipped" % target)
elif os.path.ismount(origin) and os.path.isdir(target):
arrow("%s -> %s" % (origin, target), 1)
try:
check_call(["mount", "--bind", origin, target], close_fds=True)
except CalledProcessError as e:
warn("Mount failed: %s.\n" % e)
arrow("Cheating")
# check path is a kind of linux FHS
if not os.path.exists(os.path.join(path, "etc")) or not os.path.exists(os.path.join(path, "usr")):
return
# trick resolv.conf
if os.path.exists("/etc/resolv.conf"):
arrow("resolv.conf", 1)
resolv_path = os.path.join(path, "etc", "resolv.conf")
if os.path.exists(resolv_path):
os.rename(resolv_path, "%s.isbackup" % resolv_path)
shutil.copy("/etc/resolv.conf", resolv_path)
except Exception as e:
warn("resolv.conf tricks fail: %s" % e)
# trick mtab
try:
mtab_path = os.path.join(path, "etc", "mtab")
arrow("mtab", 1)
if os.path.exists(mtab_path):
os.rename(mtab_path, "%s.isbackup" % mtab_path)
os.symlink("/proc/self/mounts", mtab_path)
except Exception as e:
warn("mtab tricks fail: %s" % e)
# try to guest distro
distro = guess_distro(path)
# in case of debian disable policy
if distro == "debian":
try: open(os.path.join(path, "etc", "debian_chroot"), "w").write("CHROOT")
# fake policy-rc.d. It must exit 101, it's an expected exitcode.
policy_path = os.path.join(path, "usr", "sbin", "policy-rc.d")
try: open(policy_path, "w").write("#!/bin/bash\nexit 101\n")
# policy-rc.d needs to be executable
chrights(policy_path, mode=0755)
def unprepare_chroot(path, mount=True):
'''
Rollback preparation of a chroot environment inside a directory
'''
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
arrow("Uncheating")
# check path is a kind of linux FHS
if os.path.exists(os.path.join(path, "etc")) and os.path.exists(os.path.join(path, "usr")):
# untrick mtab
mtab_path = os.path.join(path, "etc", "mtab")
arrow("mtab", 1)
if os.path.exists(mtab_path):
os.unlink(mtab_path)
if os.path.exists("%s.isbackup" % mtab_path):
os.rename("%s.isbackup" % mtab_path, mtab_path)
# untrick resolv.conf
if os.path.exists("/etc/resolv.conf"):
arrow("resolv.conf", 1)
resolv_path = os.path.join(path, "etc", "resolv.conf")
if os.path.exists(resolv_path):
os.unlink(resolv_path)
if os.path.exists("%s.isbackup" % resolv_path):
os.rename("%s.isbackup" % resolv_path, resolv_path)
# try to guest distro
distro = guess_distro(path)
# cleaning debian stuff
if distro == "debian":
arrow("Debian specific", 1)
for f in ("etc/debian_chroot", "usr/sbin/policy-rc.d"):
try: os.unlink(os.path.join(path, f))
except: pass
mps = ("proc", "sys", "dev", "dev/pts", "dev/shm")
arrow("Unmouting filesystems")
for mp in reversed(mps):
target = os.path.join(path, mp)
if os.path.ismount(target):
arrow(target, 1)
call(["umount", target], close_fds=True)
def chroot(path, shell="/bin/bash", mount=True):
'''
Chroot inside a directory and call shell
if mount is true, mount /{proc,dev,sys} inside the chroot
'''
# prepare to chroot
prepare_chroot(path, mount)
# chrooting
arrow("Chrooting inside %s and running %s" % (path, shell))
call(["chroot", path, shell], close_fds=True)
# revert preparation of chroot
unprepare_chroot(path, mount)