view autobot/process/factory.py @ 262:5cd95c967f70

add publichtmlport to autobot project
author Jeff Hammel <jhammel@mozilla.com>
date Wed, 28 Dec 2011 21:00:25 -0800
parents 44ca3474d03d
children db20e7be2576
line wrap: on
line source

"""
generic factories for autobot
"""

from autobot.steps import CreateVirtualenv
from buildbot.process.factory import BuildFactory
from buildbot.steps.shell import SetProperty
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shell import WithProperties


### utility functions; could go elsewhere

def find(*args):
  """
  returns a command to echo the found file cross-platform
  """
  args = ['[ -e "%s" ]; then echo "${PWD}/%s"' % (arg, arg) for arg in args]
  command = 'if %s; else false; fi' % '; elif '.join(args)
  return ['bash', '-c', command]

### factories

class FirefoxDownloaderFactory(BuildFactory):
  """
  factory to aid in downloading Firefox
  """

  def __init__(self, platform, base_url=None):

    # must have the os specified!
    assert platform.get('os'), "platform['os'] must be one of (linux, win, mac); you have %s" % platform

    # determine the url
    script = 'get-latest-tinderbox'
    if base_url:
      command = [script, '-u', base_url]
    else:
      command = [script]
    self.addStep(SetProperty(property='firefox_url',
                             command=command,
                             haltOnFailure=True
                             ))

    # get the filename
    def firefox_bundle_name(returncode, stdout, stderr):
      """
      extract the end part of the URL ~ the basename
      """
      return {'firefox_bundle': stdout.rsplit('/', 1)[-1]}

    self.addStep(SetProperty(command=['echo', WithProperties('%(firefox_url)s')],
                             extract_fn=firefox_bundle_name))

    # download Firefox
    self.addStep(ShellCommand(command=['wget',
                                       WithProperties('%(firefox_url)s'),
                                       '-O',
                                       WithProperties('%(firefox_bundle)s')],
                              description='download Firefox',
                              haltOnFailure=True))

    # three cases:
    # - linux has firefox in a .tar.bz2 file
    # - windows has firefox in a .zip file
    # - mac has firefox in a .dmg file [TODO]
    if platform['os'] == 'linux':
      self.addStep(ShellCommand(command=['tar', 'xjvf',
                                         WithProperties('%(firefox_bundle)s')],
                                haltOnFailure=True
                                ))
      self.addStep(SetProperty(property='firefox',
                               command=self.abspath('firefox/firefox')))
    elif platform['os'] == 'win':
      self.addStep(ShellCommand(command=['unzip', WithProperties('%(firefox_bundle)s')],
                                haltOnFailure=True))
      self.addStep(SetProperty(property='firefox',
                               command=self.abspath('firefox/firefox.exe')))
    elif platform['os'] == 'mac':
      # See:
      # http://mxr.mozilla.org/mozilla-central/source/build/package/mac_osx/unpack-diskimage
      # http://hg.mozilla.org/qa/mozmill-automation/file/default/libs/install.py
      # http://mxr.mozilla.org/mozilla/source/testing/release/minotaur/installdmg.sh
      # https://github.com/harthur/mozregression/blob/master/mozregression/mozInstall.py#L218
      # TODO: throw these into toolbox
      # ideally, the duplication of intent should be disappeared
      # probably in the short term, a python script should be written
      # as part of autobot, since currently the slaves will need this
      # In the longer term, a silly python package should be thrown on 
      # pypi that does this and all other implementations destroyed
      self.addStep(ShellCommand(command=['rm', '-rf', 'firefox-tmp']))
      self.addStep(ShellCommand(command=['mkdir', 'firefox-tmp'],
                                haltOnFailure=True))
      self.addStep(ShellCommand(command=['hdiutil', 'attach',
                                         '-verbose', '-noautoopen',
                                         WithProperties('%(firefox_bundle)s'),
                                         '-mountpoint', 'firefox-tmp'],
                                haltOnFailure=True))
      self.addStep(ShellCommand(command=['rm', '-rf', 'firefox']))
      def app_name(returncode, stdout, stderr):
        """
        extract the name of the app
        """
        return {'app_name': stdout.rsplit('/', 1)[-1]}
      self.addStep(SetProperty(command=['echo', 'firefox-tmp', '*.app'],
                               extract_fn=app_name))
      self.addStep(ShellCommand(command=['cp', '-r', 'firefox-tmp', 'firefox']))

      # it would be nice to reuse the logic from
      # https://github.com/mozautomation/mozmill/blob/master/mozrunner/mozrunner/runner.py
      # however, I can't think of a clever way of doing this right now.
      # When refactoring Mozilla, things like this should be kept in
      # mind wrt consolidating towards best practices instead of
      # copy+pasting and creating (dysfunctional, difficult to maintain)
      # code islands
      self.addStep(SetProperty(property='firefox',
                               command=self.abspath(WithProperties('firefox/%(app_name)s/Contents/MacOS/firefox-bin'))))
    else:
      raise NotImplementedError("FirefoxDownloader doesn't know how to work with your os: %s" % platform['os'])

  def abspath(self, path):
    """returns a command that will print the absolute path"""
    # TODO: could use WithProperties if needed
    # could also go not in a class, again if needed
    return ['python', '-c',
            'import os; print os.path.abspath("%s")' % path]


class SourceFactory(BuildFactory):
  """
  base class for factories with VCS sources
  """

  sources = {'git': [], 'hg': []}
  default_branches = {'git': 'master', 'hg': 'default'}

  def __init__(self, git=None, hg=None):

    BuildFactory.__init__(self)

    # override class-level defaults
    if git is not None:
      self.sources['git'] = git
    if hg is not None:
      self.sources['hg'] = hg

    # sanitize sources
    for source_type in self.sources:
      if isinstance(self.sources[source_type], basestring):
        self.sources[source_type] = self.sources[source_type].split()
      for index, source in enumerate(self.sources[source_type]):
        if isinstance(source, basestring):
          branch = None
          if '#' in source:
            source, branch = source.rsplit('#', 1)
        else:
          source, branch = source
        if branch is None:
          branch = self.default_branches.get(source_type, None)
        self.sources[source_type][index] = (source, branch)

  def checkout(self, **kwargs):
    """
    checkout all sources
    """
    # TODO: do the right thing with branches (they're currently ignored)
    # TODO: should give more fine-grained control

    # clone hg repositories
    for hg_source, branch in self.sources.get('hg', ()):
      self.addStep(ShellCommand(command=['hg', 'clone', hg_source],
                                **kwargs))
      if branch and branch != 'default':
        dirname = self.dirname('hg', hg_source)
        self.addStep(ShellCommand(command=['hg', '-R', dirname, 'checkout', branch],
                                  **kwargs))

    # clone the git repositories
    for git_source, branch in self.sources.get('git', ()):
      self.addStep(ShellCommand(command=['git', 'clone', git_source],
                                **kwargs))
      if branch and branch != 'master':
        dirname = self.dirname('git', git_source)
        self.addStep(ShellCommand(command=['git', '--git-dir', dirname + '/.git',
                                           '--work-tree', dirname,
                                           'checkout', branch],
                                  **kwargs))

  def dirname(self, type, repo):
    """
    get the directory name for a given repo
    """
    if type == 'git':
      dirname = repo.rstrip('/').rsplit('/', 1)[-1]
      if dirname.endswith('.git'):
        dirname = dirname[:-4]
      return dirname
    elif type == 'hg':
      return repo.rstrip('/').rsplit('/')[-1]
    else:
      raise NotImplementedError("Unknown repository type: %s" % type)


class VirtualenvFactory(SourceFactory):
  """
  create a virtualenv and install some python packages in it
  """

  def __init__(self, name='env', hg=None, git=None, checkout=True):
    """
    - name : of the virtualenv
    - hg: sources of python packages with setuptools setup.pys
    - git: git sources of python package
    """
    SourceFactory.__init__(self, hg=hg, git=git)

    # wipe any vestiges
    self.addStep(ShellCommand(command=['rm', '-rf', name]))

    # create a virtualenv
    self.addStep(CreateVirtualenv(name))

    # set properities related to the virtualenv:
    # - virtualenv location
    # - scripts location
    # - python location
    self.addStep(SetProperty(property='virtualenv',
                             command=['pwd'],
                             workdir='build/'+name))
    self.addStep(SetProperty(property='scripts',
                             command=find('Scripts', 'bin'),
                             workdir=WithProperties('%(virtualenv)s')))
    self.findScript('python')

    # add a source directory
    self.addStep(ShellCommand(command=['mkdir', '-p', 'src'],
                              workdir=WithProperties('%(virtualenv)s')))

    # checkout sources
    if checkout:
      self.checkout(workdir=WithProperties('%(virtualenv)s/src'),
                    haltOnFailure=True)


  def findScript(self, script):
    """
    find the name of the script cross-platform
    - script: unix-style name of the script
    """
    self.addStep(SetProperty(property=script,
                             command=find(script, script + '.exe'),
                             workdir=WithProperties('%(scripts)s')))

  def addScripts(self):
    """
    add the scripts directory to the $PATH
    """
    self.addStep(SetProperty(property='PATH',
                             command=WithProperties('echo %(scripts)s:$PATH')))

class PythonSourceFactory(VirtualenvFactory):
  """
  setup several python packages
  """

  def __init__(self, platform=None, name='env', hg=None, git=None):

    # setup the environment
    VirtualenvFactory.__init__(self, name=name, hg=hg, git=git)

    # install the packages
    packages = []
    for hg_source, branch in self.sources.get('hg', ()):
      package = self.dirname('hg', hg_source)
      packages.append(package)
    for git_source, branch in self.sources.get('git', ()):
      package = self.dirname('git', git_source)
      packages.append(package)
    for package in packages:
      self.addStep(ShellCommand(command=[WithProperties('%(python)s'), 'setup.py', 'install'],
                                workdir=WithProperties('%(virtualenv)s/src/' + package),
                                description='install ' + package,
                                haltOnFailure=True))