diff options
| -rwxr-xr-x | scripts/init-autotua-tmpdirs.sh | 2 | ||||
| -rw-r--r-- | slave/autotua/__init__.py | 26 | ||||
| -rw-r--r-- | slave/autotua/chroot/__init__.py | 50 | ||||
| -rw-r--r-- | slave/autotua/config.py | 71 | ||||
| -rw-r--r-- | slave/autotua/const.py | 29 | ||||
| -rw-r--r-- | slave/autotua/daemon/__init__.py | 4 | ||||
| -rw-r--r-- | slave/autotua/fetch/__init__.py | 6 | ||||
| -rw-r--r-- | slave/autotua/jobuild/__init__.py | 10 | ||||
| -rw-r--r-- | slave/autotua/sync/__init__.py | 9 | ||||
| -rw-r--r-- | slave/config/slave.cfg | 14 | ||||
| -rwxr-xr-x | slave/scripts/git-proxy-cmd.sh (renamed from scripts/git-proxy-cmd.sh) | 0 | ||||
| -rw-r--r-- | slave/setup.py | 43 | ||||
| -rwxr-xr-x | slave/test_modules.py | 44 |
13 files changed, 194 insertions, 114 deletions
diff --git a/scripts/init-autotua-tmpdirs.sh b/scripts/init-autotua-tmpdirs.sh deleted file mode 100755 index 8cb60c5..0000000 --- a/scripts/init-autotua-tmpdirs.sh +++ /dev/null @@ -1,2 +0,0 @@ -mkdir -p /var/tmp/autotua/{chroots/{pristine,updated},jobtage,tarballs/{jobfiles,stages},work} -: > /var/log/autotua-slave.log diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py index 601937d..7d5ef6c 100644 --- a/slave/autotua/__init__.py +++ b/slave/autotua/__init__.py @@ -9,7 +9,7 @@ import os, shutil, urllib2, atexit import os.path as osp import cPickle as pickle -from autotua import fetch, const, sync, chroot, jobuild +from autotua import fetch, config, sync, chroot, jobuild class Jobs: """Interface to jobs on the master server that we can do""" @@ -23,7 +23,7 @@ class Jobs: (skeleton code atm) """ jobs = [] - job_list = pickle.load(urllib2.urlopen(const.AUTOTUA_MASTER+'/slave_api/jobs')) + job_list = pickle.load(urllib2.urlopen(config.AUTOTUA_MASTER+'/slave_api/jobs')) for job_data in job_list: jobs.append(Job(job_data)) return jobs @@ -36,7 +36,7 @@ class Job: self.maint = job_data['maintainer']['username'] self.name = job_data['name'] self.stage = fetch.Fetchable(uri=[job_data['stage']]) - self.jobdir = osp.join(const.WORKDIR, self.maint, self.name) + self.jobdir = osp.join(config.WORKDIR, self.maint, self.name) self.jobtagedir = osp.join(self.jobdir, 'jobtage') self.jobtagerev = job_data['jobtagerev'] self.atoms = job_data['atoms'] @@ -55,7 +55,7 @@ class Job: def _setup_jobfiles(self): job_src = [] action = 'link' - fetcher = fetch.Fetcher(const.JOBFILE_DIR) + fetcher = fetch.Fetcher(config.JOBFILE_DIR) processor = jobuild.Processor(None, self.chroot) for jbld in self.jobuilds: processor.jobuild = jbld @@ -63,7 +63,7 @@ class Job: for i in src_uri: job_src.append(fetch.Fetchable(uri=[i])) src = fetcher.tarballdir+'/%s' - dest = self.chroot.chrootdir+const.CHAUTOTUA_DIR+'/jobfiles/%s' + dest = self.chroot.chrootdir+config.CHAUTOTUA_DIR+'/jobfiles/%s' for fetchable in job_src: fetcher.fetch(fetchable) src = src % fetchable.filename @@ -81,14 +81,14 @@ class Job: # Job metadata stuff ## Get stage3 (if required) print 'Fetching stage...' - fetcher = fetch.Fetcher(const.STAGE_DIR) + fetcher = fetch.Fetcher(config.STAGE_DIR) fetcher.fetch(self.stage) # Sync jobtage tree print 'Syncing jobtage tree...' sync.Syncer().sync() # Export from local jobtage tree print 'Exporting jobtage tree...' - sync.Syncer(uri=const.JOBTAGE_DIR, destdir=self.jobtagedir, + sync.Syncer(uri=config.JOBTAGE_DIR, destdir=self.jobtagedir, rev=self.jobtagerev, scheme="git-export").sync() ## Read config, get portage snapshot if required #self._fetch_portage_snapshot() @@ -119,4 +119,14 @@ class Job: # Tidy up before cleaning self.tidy() shutil.rmtree(self.jobdir) - os.removedirs(osp.join(const.WORKDIR, self.maint)) + os.removedirs(osp.join(config.WORKDIR, self.maint)) + +if __name__ == "__main__": + job = Jobs().getjobs()[0] + job.fetch() + if os.getuid() == 0: + job.prepare() + job.run() + job.tidy() + else: + print 'You need to be root to run job.prepare(), job.run() and job.tidy()' diff --git a/slave/autotua/chroot/__init__.py b/slave/autotua/chroot/__init__.py index 042b44d..07638b2 100644 --- a/slave/autotua/chroot/__init__.py +++ b/slave/autotua/chroot/__init__.py @@ -8,7 +8,7 @@ import os, shutil, subprocess, re import os.path as osp -from .. import const, sync +from .. import config, sync from time import strftime class PristineChroot(object): @@ -21,9 +21,9 @@ class PristineChroot(object): @type stage: string """ # stage-my.tar.bz2 -> stage-my - self.dir = osp.join(const.CHROOT_DIR, stage_filename[:stage_filename.rfind('.tar')]) + self.dir = osp.join(config.CHROOT_DIR, stage_filename[:stage_filename.rfind('.tar')]) self.stage = stage_filename - self.tar_args = (osp.join(const.STAGE_DIR, self.stage), self.dir) + self.tar_args = (osp.join(config.STAGE_DIR, self.stage), self.dir) def _extract_stage(self): os.makedirs(self.dir) @@ -33,17 +33,17 @@ class PristineChroot(object): def check(self, fix=False): """Verify that the pristine is intact""" - if const.VERBOSE: + if config.VERBOSE: print "Cross-checking with stage tarball \"%s\"" % self.stage process = subprocess.Popen('tar jfd "%s" -C "%s"' % self.tar_args, stderr=subprocess.PIPE, shell=True) - if const.VERBOSE: + if config.VERBOSE: print "PID: %s" % process.pid output = process.communicate()[1] if output.count('No such file or directory') > 0: print "Chroot is no longer pristine." regex = re.compile(r'tar: (.*?):.*') output = regex.sub(r'"\1"', output) - if const.VERBOSE: + if config.VERBOSE: print "The following files are missing:" print output if not fix: @@ -59,7 +59,7 @@ class PristineChroot(object): if osp.exists(self.dir): print "Pristine ready." return True - if const.VERBOSE: + if config.VERBOSE: print "Extracting from stage...", self._extract_stage() print "Pristine ready." @@ -67,10 +67,10 @@ class PristineChroot(object): def reset(self): """Re-extract the chroot from the tarball""" - if const.VERBOSE: + if config.VERBOSE: print "Removing existing chroot..." shutil.rmtree(self.dir) - if const.VERBOSE: + if config.VERBOSE: print "Extracting %s" % self.stage self._extract_stage() print "Pristine ready." @@ -115,18 +115,18 @@ class WorkChroot(object): for dir in ['/dev', '/sys', '/proc']: self._bind(dir, dir) # FIXME: else: Fetch portage tarball and use. - if const.PORTAGE_DIR: - if not osp.isdir(const.PORTAGE_DIR): - print "\"%s\" is not a directory, cannot mount" % const.PORTAGE_DIR + if config.PORTAGE_DIR: + if not osp.isdir(config.PORTAGE_DIR): + print "\"%s\" is not a directory, cannot mount" % config.PORTAGE_DIR else: - self._bind(const.PORTAGE_DIR, '/usr/portage') - if const.DISTFILES_DIR and not os.path.samefile(const.DISTFILES_DIR, const.PORTAGE_DIR+'/distfiles'): - if not osp.isdir(const.DISTFILES_DIR): - print "\"%s\" is not a directory, cannot mount" % const.DISTFILES_DIR + self._bind(config.PORTAGE_DIR, '/usr/portage') + if config.DISTFILES_DIR and not os.path.samefile(config.DISTFILES_DIR, config.PORTAGE_DIR+'/distfiles'): + if not osp.isdir(config.DISTFILES_DIR): + print "\"%s\" is not a directory, cannot mount" % config.DISTFILES_DIR else: - self._bind(const.DISTFILES_DIR, '/usr/portage/distfiles', ro=False) - self._bind(const.AUTOTUA_DIR+'/bin', const.CHAUTOTUA_DIR+'/bin') - self._bind(self.jobdir+'/jobtage', const.CHAUTOTUA_DIR+'/jobtage') + self._bind(config.DISTFILES_DIR, '/usr/portage/distfiles', ro=False) + self._bind(config.AUTOTUA_DIR+'/bin', config.CHAUTOTUA_DIR+'/bin') + self._bind(self.jobdir+'/jobtage', config.CHAUTOTUA_DIR+'/jobtage') def setup(self): """ @@ -142,7 +142,7 @@ class WorkChroot(object): print "Preparing Work Chroot..." sync.Syncer(uri=self.pristine.dir+"/", destdir=self.chrootdir, scheme='rsync-nc').sync() for dir in ['bin', 'jobfiles', 'jobtage', 'src']: - os.makedirs('%s/%s/%s' % (self.chrootdir, const.CHAUTOTUA_DIR, dir)) + os.makedirs('%s/%s/%s' % (self.chrootdir, config.CHAUTOTUA_DIR, dir)) self._setup_mounts() print "Work Chroot ready." @@ -151,8 +151,8 @@ class WorkChroot(object): Cleanup when done. """ self._clean_mounts() - if osp.isdir(self.chrootdir+const.CHAUTOTUA_DIR): - if not osp.isdir(self.chrootdir+const.CHAUTOTUA_DIR+'-old'): - os.makedirs(self.chrootdir+const.CHAUTOTUA_DIR+'-old') - shutil.move(self.chrootdir+const.CHAUTOTUA_DIR, - self.chrootdir+const.CHAUTOTUA_DIR+'-old/autotua-'+strftime('%Y%m%d%H%M%S')) + if osp.isdir(self.chrootdir+config.CHAUTOTUA_DIR): + if not osp.isdir(self.chrootdir+config.CHAUTOTUA_DIR+'-old'): + os.makedirs(self.chrootdir+config.CHAUTOTUA_DIR+'-old') + shutil.move(self.chrootdir+config.CHAUTOTUA_DIR, + self.chrootdir+config.CHAUTOTUA_DIR+'-old/autotua-'+strftime('%Y%m%d%H%M%S')) diff --git a/slave/autotua/config.py b/slave/autotua/config.py new file mode 100644 index 0000000..63a5209 --- /dev/null +++ b/slave/autotua/config.py @@ -0,0 +1,71 @@ +# vim: set sw=4 sts=4 et : +# Copyright: 2008 Gentoo Foundation +# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com> +# License: GPL-2 +# +# Immortal lh! +# + +""" +Default configuration, overriden by /etc/autotua/slave.cfg +""" + +import os, ConfigParser, const + +VERBOSE = False +FULL_CLEAN = False +IGNORE_PROXY = False + +LOGFILE = '/var/log/autotua/slave.log' +TMPDIR = '/var/tmp/autotua' + +AUTOTUA_MASTER = 'http://www.autotua.org:8000' +JOBTAGE_URI = 'git://git.overlays.gentoo.org/proj/jobtage.git' + +# Bind mounted inside the chroots for use if defined +PORTAGE_DIR = '/usr/portage' +DISTFILES_DIR = '/usr/portage/distfiles' + +# Read settings from slave.cfg which override the above +options = locals().copy() +cfg = ConfigParser.ConfigParser() +cfg.readfp(open('%s/slave.cfg' % const.CONFIG_PATH)) +for option, value in options.iteritems(): + if not isinstance(value, (str, int, bool)): + continue + if cfg.has_option('global', option.lower()): + if isinstance(value, str): + if not option.startswith('__'): + exec('%s = %s' % (option, cfg.get('global', option.lower()))) + elif isinstance(value, bool): + exec('%s = %s' % (option, cfg.getboolean('global', option.lower()))) + elif isinstance(value, int): + exec('%s = %s' % (option, cfg.getint('global', option.lower()))) + +# Import all the constants now +from const import * + +# Derivative variables +TARBALL_DIR = TMPDIR+'/tarballs' +STAGE_DIR = TARBALL_DIR+'/stages' +JOBFILE_DIR = TARBALL_DIR+'/jobfiles' +CHROOT_DIR = TMPDIR+'/chroots/pristine' +SAVED_CHROOT_DIR = TMPDIR+'/chroots/saved' +WORKDIR = TMPDIR+'/work' +JOBTAGE_DIR = TMPDIR+'/jobtage' + +# If using http_proxy, enable git proxy support +if not IGNORE_PROXY and os.environ.has_key('http_proxy'): + prefix = AUTOTUA_DIR+'/../scripts/' + # If git-proxy-cmd.sh is in PATH, use that + for path in os.environ['PATH'].split(':'): + if os.access(path+"/git-proxy-cmd.sh", os.X_OK): + prefix = '' + os.environ['GIT_PROXY_COMMAND'] = prefix+'git-proxy-cmd.sh' + +# Create autotua directories if they don't exist +if not os.path.isdir(os.path.dirname(LOGFILE)): + os.mkdir(os.path.dirname(LOGFILE)) +for dir in [TMPDIR, TARBALL_DIR, STAGE_DIR, JOBFILE_DIR, CHROOT_DIR, SAVED_CHROOT_DIR, WORKDIR, JOBTAGE_DIR]: + if not os.path.isdir(dir): + os.mkdir(dir) diff --git a/slave/autotua/const.py b/slave/autotua/const.py index 7d23e1f..dd7b969 100644 --- a/slave/autotua/const.py +++ b/slave/autotua/const.py @@ -12,35 +12,8 @@ Internal Constants import os -VERBOSE = True -FULL_CLEAN = False -USE_PROXY = True - +CONFIG_PATH = '/etc/autotua' AUTOTUA_DIR = os.path.abspath(os.path.dirname(__file__)) -LOGFILE = '/var/log/autotua-slave.log' -TMPDIR = '/var/tmp/autotua' -TARBALL_DIR = TMPDIR+'/tarballs' -STAGE_DIR = TARBALL_DIR+'/stages' -JOBFILE_DIR = TARBALL_DIR+'/jobfiles' -CHROOT_DIR = TMPDIR+'/chroots/pristine' -WORKDIR = TMPDIR+'/work' -JOBTAGE_DIR = TMPDIR+'/jobtage' - -AUTOTUA_MASTER = 'http://www.autotua.org:8000' -JOBTAGE_URI = 'git://git.overlays.gentoo.org/proj/jobtage.git' # Autotua variables inside the chroot CHAUTOTUA_DIR = '/tmp/autotua' - -# Bind mounted inside the chroots for use if defined -PORTAGE_DIR = '' -DISTFILES_DIR = '' - -# If using http_proxy, enable git proxy support -if os.environ.has_key('http_proxy'): - prefix = AUTOTUA_DIR+'/../../scripts/' - # If git-proxy-cmd.sh is in PATH, use that - for path in os.environ['PATH'].split(':'): - if os.access(path+"/git-proxy-cmd.sh", os.X_OK): - prefix = '' - os.environ['GIT_PROXY_COMMAND'] = prefix+'git-proxy-cmd.sh' diff --git a/slave/autotua/daemon/__init__.py b/slave/autotua/daemon/__init__.py index 6d42817..a5e4f86 100644 --- a/slave/autotua/daemon/__init__.py +++ b/slave/autotua/daemon/__init__.py @@ -7,7 +7,7 @@ # import subprocess -from .. import const +from .. import config class Spawn(object): """ @@ -16,7 +16,7 @@ class Spawn(object): sent to stdin with a 'pong\n' """ - def __init__(self, chroot, jobtagedir, workdir, command, logfile=const.LOGFILE): + def __init__(self, chroot, jobtagedir, workdir, command, logfile=config.LOGFILE): """ @param chroot: The path to the chroot in which the command will be run diff --git a/slave/autotua/fetch/__init__.py b/slave/autotua/fetch/__init__.py index 36588d6..224f056 100644 --- a/slave/autotua/fetch/__init__.py +++ b/slave/autotua/fetch/__init__.py @@ -65,3 +65,9 @@ class Fetcher(object): while index >= 0: subprocess.check_call(self.command % { 'file': filename, 'uri': target.uri[index], 'tries': 5 }, shell=True) index -= 1 + +if __name__ == "__main__": + import autotua + job = autotua.Jobs().getjobs()[0] + fetcher = Fetcher(tmpdir) + fetcher.fetch(job.stage) diff --git a/slave/autotua/jobuild/__init__.py b/slave/autotua/jobuild/__init__.py index a250b66..54d3ca0 100644 --- a/slave/autotua/jobuild/__init__.py +++ b/slave/autotua/jobuild/__init__.py @@ -11,7 +11,7 @@ import re, subprocess, sys, os import os.path as osp -from .. import const, daemon +from .. import config, daemon class Jobuild(object): """A jobuild""" @@ -129,10 +129,10 @@ class Processor(object): def _msg(self, msg): chroot = None - workdir = const.AUTOTUA_DIR + workdir = config.AUTOTUA_DIR if self.chroot: chroot = self.chroot.chrootdir - workdir = const.CHAUTOTUA_DIR + workdir = config.CHAUTOTUA_DIR proc = daemon.Spawn(chroot, self.jobuild.jobtagedir, workdir, '\"%s\"/bin/jobuild.sh %s' % (workdir, msg)) # XXX: This waits for process to close stderr. # Replace with a loop during daemonisation @@ -177,6 +177,10 @@ class Resolver(object): return newlist def _unique(self, deplist): + """ + Makes deplist unique and reverses order + This gives us the required merge order + """ newlist = [] index = len(deplist) while index >= 0: diff --git a/slave/autotua/sync/__init__.py b/slave/autotua/sync/__init__.py index 97ca095..7ad356f 100644 --- a/slave/autotua/sync/__init__.py +++ b/slave/autotua/sync/__init__.py @@ -8,7 +8,7 @@ import os, subprocess, shutil import os.path as osp -from .. import const +from .. import config # init == sync (example: rsync) ONLY_SYNC = 1 @@ -23,7 +23,7 @@ class Syncer(object): Sync stuff """ - def __init__(self, scheme='git', uri=const.JOBTAGE_URI, rev=None, destdir=const.JOBTAGE_DIR): + def __init__(self, scheme='git', uri=config.JOBTAGE_URI, rev=None, destdir=config.JOBTAGE_DIR): """ Default is to sync the jobtage tree from the default location. @@ -143,3 +143,8 @@ class Command(object): """Run a sync command""" args = self._get_args(action=action) subprocess.check_call(args, shell=True) + +if __name__ == "__main__": + syncer = Syncer(destdir='/tmp/jobtage') + syncer.sync() + diff --git a/slave/config/slave.cfg b/slave/config/slave.cfg new file mode 100644 index 0000000..ca17132 --- /dev/null +++ b/slave/config/slave.cfg @@ -0,0 +1,14 @@ +# AutotuA Slave configuration +# Default values are commented +# No trailing slashes please + +[global] +verbose = True +;full_clean = False +;ignore_proxy = False +;logfile = '/var/log/autotua/slave.log' +;tmpdir = '/var/tmp/autotua' +;autotua_master = 'http://www.autotua.org:8000' +;jobtage_uri = 'git://git.overlays.gentoo.org/proj/jobtage.git' +;portage_dir = '/usr/portage' +;distfiles_dir = '/usr/portage/distfiles' diff --git a/scripts/git-proxy-cmd.sh b/slave/scripts/git-proxy-cmd.sh index aa9fc61..aa9fc61 100755 --- a/scripts/git-proxy-cmd.sh +++ b/slave/scripts/git-proxy-cmd.sh diff --git a/slave/setup.py b/slave/setup.py new file mode 100644 index 0000000..a9c0bd5 --- /dev/null +++ b/slave/setup.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# vim: set sw=4 sts=4 et : +# Copyright: 2008 Gentoo Foundation +# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com> +# License: GPL-2 +# +# Immortal lh! +# + +import os +from distutils.core import Command, setup + +class print_help(Command): + description = "Print help" + user_options = [('help','h',"Help help")] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + print \ + """ + setup.py: Run this file as root to install the AutotuA slave + * Edit /etc/autotua/slave.cfg for the settings + * Set the http_proxy variable if you want to use an http proxy (will tunnel everything) + - You can force autotua to not use the http_proxy for tunnelling by setting ignore_proxy = true + """ + +setup(name='autotua-slave', + version='0.0.1', + description='Build slave for AutotuA', + author='Nirbheek Chauhan', + author_email='nirbheek.chauhan@gmail.com', + license='GPL-2', + url='http://www.autotua.org/', + cmdclass={'help': print_help}, + data_files=[('/etc/autotua', ['config/slave.cfg']), ('/usr/bin', ['scripts/git-proxy-cmd.sh'])], + packages=['autotua', 'autotua.chroot', 'autotua.daemon', 'autotua.fetch', 'autotua.jobuild', 'autotua.sync'], + package_data={'autotua': ['bin/*.sh']}, + ) diff --git a/slave/test_modules.py b/slave/test_modules.py deleted file mode 100755 index 34013a1..0000000 --- a/slave/test_modules.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# vim: set sw=4 sts=4 et : -# Copyright: 2008 Gentoo Foundation -# Author(s): Nirbheek Chauhan <nirbheek.chauhan@gmail.com> -# License: GPL-2 -# -# Immortal lh! -# - -import autotua, shutil, os, sys -import os.path as osp -from autotua import chroot, fetch, sync - -tmpdir = '/tmp' -# Default test (tests everything) -modules = ['job'] -# All tests: -#modules = ['job', 'fetch', 'sync', 'chroot'] - -if 'fetch' in modules: - job = autotua.Jobs().getjobs()[0] - fetcher = autotua.fetch.Fetcher(tmpdir) - fetcher.fetch(job.stage) - -if 'sync' in modules: - destdir = tmpdir+'/jobtage' - syncer = autotua.sync.Syncer(destdir=destdir) - syncer.sync() - -if 'chroot' in modules: - job = autotua.Jobs().getjobs()[0] - chroot = autotua.chroot.WorkChroot(job.jobdir, job.stage.filename) - chroot.setup() - chroot.tidy() - -if 'job' in modules: - job = autotua.Jobs().getjobs()[0] - job.fetch() - if os.getuid() == 0: - job.prepare() - job.run() - job.tidy() - else: - print 'You need to be root to run job.prepare(), job.run() and job.tidy()' |