Tabula rasa

url='https://github.com/seblu/aurbot'

license=('GPL2')

makedepends=('python-distribute')

depends=('python' 'pyalpm' 'python-aur')

depends=('python')

optdepends=('devtools')

package() {

  cd "$startdir"

#!/usr/bin/python3

# coding: utf-8

# aurbot - Archlinux User Repository Builder Bot

# Copyright © 2014 Sébastien Luttringer

#

# Started, October 30th 2011

#

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; either version 2

# of the License, or (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

from argparse import ArgumentParser

from configparser import ConfigParser

from json import load as jload, dump as jdump, loads as jloads

from logging import getLogger, debug, warning, error, DEBUG

from os import getcwd, chdir

from os.path import join

from subprocess import check_call

from tarfile import open as tar

from tempfile import TemporaryDirectory

from time import sleep

from urllib.request import urlopen, Request

from xdg.BaseDirectory import save_config_path, save_cache_path

AUR_URL = 'https://aur.archlinux.org'

USER_AGENT = "aurbot"

XDG_DIRECTORY = "aurbot"

ERR_USAGE = 1

ERR_FATAL = 2

ERR_ABORT = 3

ERR_UNKNOWN = 4

class AURPackage(dict):

	'''

	Abstract AUR package action

	'''

	def __init__(self, name, timeout=None):

		self.name = name

		debug("getting %s aur infos" % self.name)

		url = "%s/rpc.php?type=info&arg=%s" % (AUR_URL, name)

		url_req = Request(url, headers={"User-Agent": USER_AGENT})

		debug("Requesting url: %s" % url)

		debug("Timeout is %s" % timeout)

		url_fd = urlopen(url_req, timeout=timeout)

		d = jloads(url_fd.read().decode("utf-8"))

		if d["version"] != 1:

			raise Exception("Unknown AUR Backend version")

		self._info = d["results"]

	def __getattr__(self, name):

		for k, v in self._info.items():

			if name == k.lower():

				return v

		raise AttributeError()

	def __repr__(self):

		return "%s v%s (%s)" % (self.name, self.version, self.description)

	def extract(self, path):

		'''

		Extract aur source tarball inside a directory path

		'''

		fo = urlopen('%s/%s' % (AUR_URL, self.urlpath))

		tarball = tar(mode='r|*', fileobj=fo)

		tarball.extractall(path)

		fo.close()

class JsonDictFile(dict):

	'''Json serialized dict'''

	def __init__(self, path):

		'''Load json file'''

		assert(path is not None)

		try:

			self._fileobj = open(path, "a+")

		except (IOError, OSError) as exp:

			error("Unable to access to json file %s: %s" % (path, exp))

			raise

		if self._fileobj.seek(0, 1) == 0:

			debug("Json file is empty")

		else:

			debug("Loading json file %s" % path)

			try:

				self._fileobj.seek(0, 0)

				dico = jload(self._fileobj)

				self.update(dico)

			except Exception as exp:

				error("Unable to load json file %s: %s" % (path, exp))

				raise

	def __del__(self):

		'''Save current dict into a json file'''

		if len(self) == 0:

			debug("Not saved. Dict is empty")

			return

		if self._fileobj is not None:

			debug("Saving dict into json file")

			try:

				self._fileobj.seek(0, 0)

				self._fileobj.truncate(0)

				jdump(self, self._fileobj)

			except Exception as exp:

				error("Unable to save json file: %s" % exp)

				raise

def build(localpkg, aurpkg):

	debug("we build %s" % aurpkg.name)

	# find build dir

	build_dir = TemporaryDirectory()

	debug("build dir is %s" % build_dir.name)

	# extract tarball

	debug("extracting aur tarball")

	aurpkg.extract(build_dir.name)

	cwd = getcwd()

	try:

		chdir("%s/%s" % (build_dir.name, aurpkg.name))

		check_call(localpkg["build_cmd"], shell=True, close_fds=True)

		# build was succesful

		check_call(localpkg["commit_cmd"], shell=True, close_fds=True)

	finally:

		chdir(cwd)

def event_loop(packages, cache, timeout):

	'''

	program roundabout

	'''

	while True:

		for name in packages.sections():

			debug("Checking %s" % name)

			pkg = AURPackage(name)

			print(pkg)

			# For security, if the maintainer has changed we pass

			maintainer = packages[name].get("maintainer", None)

			if maintainer != pkg.maintainer:

				warning("Invalid maintainer for package %s" % name)

				debug("registered maintainer: %s" % maintainer)

				debug("AUR maintainer: %s" % pkg.maintainer)

				continue

			# checks update

			if pkg.lastmodified <= cache.get(name, 0):

				debug("%s was already built" % name)

				continue

			# package needs to be built and commited

			try:

				build(packages[name], pkg)

			except Exception as exp:

				warning("chiche: %s" % exp)

				continue

			# we save last successful build in cache

			cache[name] = pkg.lastmodified

		# night is coming

		debug("waiting for %ds" % timeout)

		sleep(timeout)

def parse_argv():

	'''Parse command line arguments'''

	# load parser

	parser = ArgumentParser()

	parser.add_argument("-p", "--packages-path", help="packages config file path")

	parser.add_argument("-c", "--cache-path", help="cache file path")

	parser.add_argument("-s", "--sleep", type=int, default=86400, help="sleep interval between checks")

	parser.add_argument("-d", "--debug", action="store_true", help="debug mode")

	# parse it!

	args = parser.parse_args()

	# set global debug mode

	if args.debug:

		getLogger().setLevel(DEBUG)

		debug("debug on")

	# set default paths

	if args.packages_path is None:

		args.packages_path = join(save_config_path(XDG_DIRECTORY), "packages.conf")

	if args.cache_path is None:

		args.cache_path = join(save_cache_path(XDG_DIRECTORY), "packages.cache")

	return args

def main():

	'''Program entry point'''

	try:

		# parse command line

		args = parse_argv()

		# parse package list

		packages = ConfigParser()

		packages.read(args.packages_path)

		# load cache

		cache = JsonDictFile(args.cache_path)

		event_loop(packages, cache, args.sleep)

	except KeyboardInterrupt:

		exit(ERR_ABORT)

	# except BaseError as exp:

	#     error(exp)

	#     exit(ERR_FATAL)

	except Exception as exp:

		error(exp)

		if getLogger().getEffectiveLevel() != DEBUG:

			error("Unknown error. Please report it with --debug.")

		else:

			raise

		exit(ERR_UNKNOWN)

if __name__ == '__main__':

	main()

 No newline at end of file

#!/usr/bin/python3

# coding: utf-8

# aurbot - Archlinux User Repository Builder Bot

# Copyright © 2014 Sébastien Luttringer

# Started, October 30th 2011

#

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; either version 2

# of the License, or (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

import os

import time

import json

import glob

import shutil

import argparse

import tempfile

import tarfile

import urllib.request

import configparser

import subprocess

import pyalpm

import AUR

class Repositories(dict):

	'''

	Repository class abstract all stuff around repositories

	'''

	def init(self, repo):

		'''

		Init a repository

		'''

		# create repository path

		if self[repo]['path'] is not None:

			os.makedirs(self[repo]['path'], exist_ok=True)

		# create chroot

		if self[repo]['chroot'] is not None:

			root = os.path.join(self[repo]['chroot'], 'root')

			if not os.path.exists(root):

				os.makedirs(self[repo]['chroot'], exist_ok=True)

				cmds = ['mkarchroot', '-n', root, 'base', 'base-devel', 'sudo']

				if self[repo]['arch'] == 'i686':

					cmds.insert(0, 'linux32')

				elif self[repo]['arch'] == 'x86_64':

					cmds.insert(0, 'linux64')

				if os.geteuid() != 0:

					cmds.insert(0, 'sudo')

				subprocess.check_call(cmds, close_fds=True)

	def load(self, conf_path, db_path):

		'''

		Load repositories configuration

		'''

		try:

			# read config

			cp = configparser.RawConfigParser()

			cp.read(conf_path)

			for sec in cp.sections():

				self[sec] = {

					'dbname': '%s.db.tar.gz' % sec,

					'arch': None,

					'path': None,

					'chroot': None,

					'build_command': None,

					'statusdb': None}

				self[sec].update(cp.items(sec))

				self[sec]['packages'] = cp.get(sec, 'packages', fallback='').split()

				# checks

				if self[sec]['arch'] not in ('x86_64', 'i686'):

					raise Exception('Invalid arch')

				if self[sec]['path'] is None:

					raise Exception('Invalid repository path')

		except Exception as e:

			print('Unable to load package database: %s' % e)

	def tobuild(self):

		'''

		List of packages to build

		'''

		for repo in self:

			for pkg in self[repo]['packages']:

				yield (repo, pkg)

	def build(self, repo, path):

		'''

		Build package package inside repo

		'''

		cwd = os.getcwd()

		try:

			# chdir inside builddir

			os.chdir(path)

			# output fd

			if verbose == 2:

				 devnull = None

			else:

				devnull = open('/dev/null', 'w')

			# define run commands

			cmds = ['makechrootpkg', '-c', '-r', self[repo]['chroot']]

			if self[repo]['arch'] == 'i686':

				cmds.insert(0, 'linux32')

			elif self[repo]['arch'] == 'x86_64':

				cmds.insert(0, 'linux64')

			if os.geteuid() != 0:

				cmds.insert(0, 'sudo')

			# run build

			subprocess.check_call(cmds, stdout=devnull, stderr=devnull, close_fds=True)

		finally:

			os.chdir(cwd)

	def add(self, repo, files=[]):

		'''

		Add pkg to repo

		'''

		dbpath = os.path.join(self[repo]['path'], self[repo]['dbname'])

		for f in files:

			dstpath = os.path.join(self[repo]['path'], os.path.basename(f))

			shutil.copy(f, dstpath)

			subprocess.check_call(['repo-add', dbpath, dstpath], close_fds=True)

	def all_packages(self):

		'''

		Return a set of all pacakges

		'''

		pkgs = set()

		for repo in self:

			for pkg in self[repo]['packages']:

				pkgs.add(pkg)

		return pkgs

class AURPackages(dict):

	'''

	Packages class handle package informations

	'''

	aur_url = 'https://aur.archlinux.org'

	aur_min_update_interval = 60

	def load(self, path):

		'''

		Load packages informations from a file

		'''

		if not os.path.exists(path):

			return

		try:

			self.update(json.load(open(path, "r")))

		except Exception as e:

			print('Unable to load package database: %s' % e)

	def save(self, path):

		'''

		Save packages informations to a file

		'''

		json.dump(self, open(path, 'w'))

	def register(self, pkg, update=False):

		if pkg not in self or update:

			self.aur_update(pkg)

	def aur_update(self, pkg):

		'''

		update information about a package

		'''

		if (pkg in self and

		    'aurbuilder_update' in self[pkg] and

		    self[pkg]['aurbuilder_update'] + self.aur_min_update_interval > time.time()):

			return

		info = AUR.aur_query('info', pkg)

		if info is not None:

			self[pkg] = info

			self[pkg]['aurbuilder_update'] = time.time()

		else:

			print('No package %s' % pkg)

	def extract(self, pkg, path):

		'''

		Extract aur source tarball inside a directory path

		'''

		# feed package db

		self.register(pkg)

		# get tarball

		fo = urllib.request.urlopen('%s/%s' % (self.aur_url, self[pkg]['URLPath']))

		# extract tarball

		tarball = tarfile.open(mode='r|*', fileobj=fo)

		tarball.extractall(path)

##################

#Printing commands

##################

def msg(message):

	if verbose > 0:

		print('\033[1;32m==>\033[m %s' % message)

def msg2(message):

	if verbose > 0:

		print('  \033[1;34m->\033[m %s' % message)

################

#Parser commands

################

def c_init(args):

	'''

	Init command

	'''

	msg('Initializing repositories')

	for repo in args.repos:

		msg2(repo)

		args.repos.init(repo)

def c_update(args):

	'''

	Update command

	'''

	msg('Updating AUR packages database')

	for pkg in sorted(args.repos.all_packages()):

		msg2(pkg)

		args.aurpkg.aur_update(pkg)

def c_build(args):

	'''

	Build command

	'''

	# start building

	for repo, pkg in args.repos.tobuild():

		try:

			msg('Building %s in %s' % (pkg, repo))

			# creating temp directory to extract tarball

			tempd = tempfile.TemporaryDirectory()

			# extract package inside tempdir

			msg2('Downloading from AUR')

			aurpkg.extract(pkg, tempd.name)

			# build package

			msg2('Compiling')

			builddir = os.path.join(tempd.name, pkg)

			args.repos.build(repo, builddir)

			files = glob.glob(os.path.join(builddir, '*.pkg.tar.xz'))

			if len(files) == 0:

				raise Exception('Unable to find binary packages')

			# add to repository

			msg2('Adding to repository')

			args.repos.add(repo, files)

		except Exception as e:

			# FIXME: mark package as invalid to build

			print('build failure: %s' % e)

# we start here

parser = argparse.ArgumentParser()

parser.add_argument('-r', '--repo-conf', default='repositories.conf', help='repository definitions')

parser.add_argument('-R', '--repo-db', default='repositories.json', help='repositories databases')

parser.add_argument('-p', '--aurpkg-db', default='packages.json', help='AUR packages database')

mg = parser.add_mutually_exclusive_group()

mg.add_argument('-q', '--quiet', action='store_true', default=False,

	help='quiet mode')

mg.add_argument('-v', '--verbose', action='store_true', default=False,

	help='verbose mode')

sp = parser.add_subparsers()

p_init = sp.add_parser('init')

p_init.set_defaults(func=c_init)

p_update = sp.add_parser('update')

p_update.set_defaults(func=c_update)

p_build = sp.add_parser('build')

p_build.set_defaults(func=c_build)

# parse args

args = parser.parse_args()

# path must be absolute!

args.repo_conf = os.path.abspath(args.repo_conf)

args.repo_db = os.path.abspath(args.repo_db)

args.aurpkg_db = os.path.abspath(args.aurpkg_db)

# set global output state

if args.verbose:

	verbose = 2

elif args.quiet:

	verbose = 0

else:

	verbose = 1

# load repo mananger

msg('Loading repositories configurations')

repos = Repositories()

repos.load(args.repo_conf, args.repo_db)

# load package manager

msg('Loading AUR pacakages database')

aurpkg = AURPackages()

aurpkg.load(args.aurpkg_db)

# add aurpkg and repos to args

args.aurpkg = aurpkg

args.repos = repos

# run command function

args.func(args)

# save aur packages db

msg('Saving AUR pacakages database')

aurpkg.save(args.aurpkg_db)

[DEFAULT]

build_cmd = seblu-build

commit_cmd = seblu-push

[virtualbox-ext-oracle]

maintainer = seblu

[chromium-pepper-flash]

maintainer = ava1ar

[ttf-ms-fonts]

maintainer = birdflesh

[ttf-vista-fonts]

maintainer = jnbek

[ttf-mac-fonts]

maintainer = Shanto

[ttf-adobe-fonts]

maintainer = Shanto

[ttf-tahoma]

maintainer = tdy

[sublime-text]

maintainer = jiggak

[bedup]

maintainer = omangold

[python-i3-git]

maintainer = Alad

force = 2592000 #30days

[debootstrap]

maintainer = dolik.rce

[iozone]

maintainer = jsst

 No newline at end of file

[DEFAULT]

build_cmd = true

commit_cmd = true

[virtualbox-ext-oracle]

maintainer = seblu

[python-i3-git]

maintainer = Alad

force = 2592000 #30days