Mercurial > hg > autobot
view autobot/process/factory.py @ 262:5cd95c967f70
add publichtmlport to autobot project
author | Jeff Hammel <jhammel@mozilla.com> |
---|---|
date | Wed, 28 Dec 2011 21:00:25 -0800 |
parents | 44ca3474d03d |
children | db20e7be2576 |
line wrap: on
line source
""" generic factories for autobot """ from autobot.steps import CreateVirtualenv from buildbot.process.factory import BuildFactory from buildbot.steps.shell import SetProperty from buildbot.steps.shell import ShellCommand from buildbot.steps.shell import WithProperties ### utility functions; could go elsewhere def find(*args): """ returns a command to echo the found file cross-platform """ args = ['[ -e "%s" ]; then echo "${PWD}/%s"' % (arg, arg) for arg in args] command = 'if %s; else false; fi' % '; elif '.join(args) return ['bash', '-c', command] ### factories class FirefoxDownloaderFactory(BuildFactory): """ factory to aid in downloading Firefox """ def __init__(self, platform, base_url=None): # must have the os specified! assert platform.get('os'), "platform['os'] must be one of (linux, win, mac); you have %s" % platform # determine the url script = 'get-latest-tinderbox' if base_url: command = [script, '-u', base_url] else: command = [script] self.addStep(SetProperty(property='firefox_url', command=command, haltOnFailure=True )) # get the filename def firefox_bundle_name(returncode, stdout, stderr): """ extract the end part of the URL ~ the basename """ return {'firefox_bundle': stdout.rsplit('/', 1)[-1]} self.addStep(SetProperty(command=['echo', WithProperties('%(firefox_url)s')], extract_fn=firefox_bundle_name)) # download Firefox self.addStep(ShellCommand(command=['wget', WithProperties('%(firefox_url)s'), '-O', WithProperties('%(firefox_bundle)s')], description='download Firefox', haltOnFailure=True)) # three cases: # - linux has firefox in a .tar.bz2 file # - windows has firefox in a .zip file # - mac has firefox in a .dmg file [TODO] if platform['os'] == 'linux': self.addStep(ShellCommand(command=['tar', 'xjvf', WithProperties('%(firefox_bundle)s')], haltOnFailure=True )) self.addStep(SetProperty(property='firefox', command=self.abspath('firefox/firefox'))) elif platform['os'] == 'win': self.addStep(ShellCommand(command=['unzip', WithProperties('%(firefox_bundle)s')], haltOnFailure=True)) self.addStep(SetProperty(property='firefox', command=self.abspath('firefox/firefox.exe'))) elif platform['os'] == 'mac': # See: # http://mxr.mozilla.org/mozilla-central/source/build/package/mac_osx/unpack-diskimage # http://hg.mozilla.org/qa/mozmill-automation/file/default/libs/install.py # http://mxr.mozilla.org/mozilla/source/testing/release/minotaur/installdmg.sh # https://github.com/harthur/mozregression/blob/master/mozregression/mozInstall.py#L218 # TODO: throw these into toolbox # ideally, the duplication of intent should be disappeared # probably in the short term, a python script should be written # as part of autobot, since currently the slaves will need this # In the longer term, a silly python package should be thrown on # pypi that does this and all other implementations destroyed self.addStep(ShellCommand(command=['rm', '-rf', 'firefox-tmp'])) self.addStep(ShellCommand(command=['mkdir', 'firefox-tmp'], haltOnFailure=True)) self.addStep(ShellCommand(command=['hdiutil', 'attach', '-verbose', '-noautoopen', WithProperties('%(firefox_bundle)s'), '-mountpoint', 'firefox-tmp'], haltOnFailure=True)) self.addStep(ShellCommand(command=['rm', '-rf', 'firefox'])) def app_name(returncode, stdout, stderr): """ extract the name of the app """ return {'app_name': stdout.rsplit('/', 1)[-1]} self.addStep(SetProperty(command=['echo', 'firefox-tmp', '*.app'], extract_fn=app_name)) self.addStep(ShellCommand(command=['cp', '-r', 'firefox-tmp', 'firefox'])) # it would be nice to reuse the logic from # https://github.com/mozautomation/mozmill/blob/master/mozrunner/mozrunner/runner.py # however, I can't think of a clever way of doing this right now. # When refactoring Mozilla, things like this should be kept in # mind wrt consolidating towards best practices instead of # copy+pasting and creating (dysfunctional, difficult to maintain) # code islands self.addStep(SetProperty(property='firefox', command=self.abspath(WithProperties('firefox/%(app_name)s/Contents/MacOS/firefox-bin')))) else: raise NotImplementedError("FirefoxDownloader doesn't know how to work with your os: %s" % platform['os']) def abspath(self, path): """returns a command that will print the absolute path""" # TODO: could use WithProperties if needed # could also go not in a class, again if needed return ['python', '-c', 'import os; print os.path.abspath("%s")' % path] class SourceFactory(BuildFactory): """ base class for factories with VCS sources """ sources = {'git': [], 'hg': []} default_branches = {'git': 'master', 'hg': 'default'} def __init__(self, git=None, hg=None): BuildFactory.__init__(self) # override class-level defaults if git is not None: self.sources['git'] = git if hg is not None: self.sources['hg'] = hg # sanitize sources for source_type in self.sources: if isinstance(self.sources[source_type], basestring): self.sources[source_type] = self.sources[source_type].split() for index, source in enumerate(self.sources[source_type]): if isinstance(source, basestring): branch = None if '#' in source: source, branch = source.rsplit('#', 1) else: source, branch = source if branch is None: branch = self.default_branches.get(source_type, None) self.sources[source_type][index] = (source, branch) def checkout(self, **kwargs): """ checkout all sources """ # TODO: do the right thing with branches (they're currently ignored) # TODO: should give more fine-grained control # clone hg repositories for hg_source, branch in self.sources.get('hg', ()): self.addStep(ShellCommand(command=['hg', 'clone', hg_source], **kwargs)) if branch and branch != 'default': dirname = self.dirname('hg', hg_source) self.addStep(ShellCommand(command=['hg', '-R', dirname, 'checkout', branch], **kwargs)) # clone the git repositories for git_source, branch in self.sources.get('git', ()): self.addStep(ShellCommand(command=['git', 'clone', git_source], **kwargs)) if branch and branch != 'master': dirname = self.dirname('git', git_source) self.addStep(ShellCommand(command=['git', '--git-dir', dirname + '/.git', '--work-tree', dirname, 'checkout', branch], **kwargs)) def dirname(self, type, repo): """ get the directory name for a given repo """ if type == 'git': dirname = repo.rstrip('/').rsplit('/', 1)[-1] if dirname.endswith('.git'): dirname = dirname[:-4] return dirname elif type == 'hg': return repo.rstrip('/').rsplit('/')[-1] else: raise NotImplementedError("Unknown repository type: %s" % type) class VirtualenvFactory(SourceFactory): """ create a virtualenv and install some python packages in it """ def __init__(self, name='env', hg=None, git=None, checkout=True): """ - name : of the virtualenv - hg: sources of python packages with setuptools setup.pys - git: git sources of python package """ SourceFactory.__init__(self, hg=hg, git=git) # wipe any vestiges self.addStep(ShellCommand(command=['rm', '-rf', name])) # create a virtualenv self.addStep(CreateVirtualenv(name)) # set properities related to the virtualenv: # - virtualenv location # - scripts location # - python location self.addStep(SetProperty(property='virtualenv', command=['pwd'], workdir='build/'+name)) self.addStep(SetProperty(property='scripts', command=find('Scripts', 'bin'), workdir=WithProperties('%(virtualenv)s'))) self.findScript('python') # add a source directory self.addStep(ShellCommand(command=['mkdir', '-p', 'src'], workdir=WithProperties('%(virtualenv)s'))) # checkout sources if checkout: self.checkout(workdir=WithProperties('%(virtualenv)s/src'), haltOnFailure=True) def findScript(self, script): """ find the name of the script cross-platform - script: unix-style name of the script """ self.addStep(SetProperty(property=script, command=find(script, script + '.exe'), workdir=WithProperties('%(scripts)s'))) def addScripts(self): """ add the scripts directory to the $PATH """ self.addStep(SetProperty(property='PATH', command=WithProperties('echo %(scripts)s:$PATH'))) class PythonSourceFactory(VirtualenvFactory): """ setup several python packages """ def __init__(self, platform=None, name='env', hg=None, git=None): # setup the environment VirtualenvFactory.__init__(self, name=name, hg=hg, git=git) # install the packages packages = [] for hg_source, branch in self.sources.get('hg', ()): package = self.dirname('hg', hg_source) packages.append(package) for git_source, branch in self.sources.get('git', ()): package = self.dirname('git', git_source) packages.append(package) for package in packages: self.addStep(ShellCommand(command=[WithProperties('%(python)s'), 'setup.py', 'install'], workdir=WithProperties('%(virtualenv)s/src/' + package), description='install ' + package, haltOnFailure=True))