# HG changeset patch # User Jeff Hammel # Date 1295384518 28800 # Node ID d80e96f7e54786ca779953d7ce32e9a87f9ba1aa # Parent 34b1d30503fad7577e13c1f2210b92396965f3e4 flush out hgpoller; untested diff -r 34b1d30503fa -r d80e96f7e547 autobot/changes/poller.py --- a/autobot/changes/poller.py Fri Jan 14 08:06:00 2011 -0800 +++ b/autobot/changes/poller.py Tue Jan 18 13:01:58 2011 -0800 @@ -13,9 +13,9 @@ # # Copyright Buildbot Team Members +import os import time import tempfile -import os from twisted.python import log from twisted.internet import defer, utils @@ -23,8 +23,10 @@ from buildbot.changes import base class Poller(base.PollingChangeSource): - """This source will poll a remote git repo for changes and submit - them to the change master.""" + """ + This will poll a remote resource for changes and submit + them to the change master. + """ compare_attrs = ["repourl", "branch", "workdir", "pollInterval", "binary", "usetimestamps", @@ -32,14 +34,15 @@ def __init__(self, repourl, binary=None, branch=None, workdir=None, pollInterval=10*60, - binary='git', usetimestamps=True, + binary=None, usetimestamps=True, category=None, project=None, pollinterval=-2): + # for backward compatibility; the parameter used to be spelled with 'i' if pollinterval != -2: pollInterval = pollinterval + if project is None: project = '' - self.repourl = repourl self.branch = branch self.pollInterval = pollInterval @@ -54,12 +57,13 @@ self.commitInfo = {} self.initLock = defer.DeferredLock() - if self.workdir == None: + if not self.workdir: self.workdir = tempfile.gettempdir() self.name = self.__class__.__name__ def startService(self): + # initialize the repository we'll use to get changes; note that # startService is not an event-driven method, so this method will # instead acquire self.initLock immediately when it is called. @@ -74,6 +78,9 @@ @deferredLocked('initLock') def initRepository(self): + """initialize a repository or whatever""" + + # make the directory, if necessary d = defer.succeed(None) def make_dir(_): dirpath = os.path.dirname(self.workdir.rstrip(os.sep)) @@ -82,16 +89,17 @@ os.makedirs(dirpath) d.addCallback(make_dir) + # perform the initialization def processCommand(command): - d = utils.getProcessOutputAndValue(self.gitbin, + d = utils.getProcessOutputAndValue(self.binary, command, env=dict(PATH=os.environ['PATH'])) d.addCallback(self._convert_nonzero_to_failure) d.addErrback(self._stop_on_failure) - return d - + return d for command in self.initializationCommands(): d.addCallback(lambda _: processCommand(command[:])) + # finish up def log_finished(_): log.msg("%s: finished initializing working dir from %s" % (self.name, self.repourl) d.addCallback(log_finished) @@ -113,60 +121,13 @@ d.addCallback(self._catch_up) d.addErrback(self._catch_up_failure) return d - - def _get_commit_comments(self, rev): - args = ['log', rev, '--no-walk', r'--format=%s%n%b'] - d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) - def process(git_output): - stripped_output = git_output.strip() - if len(stripped_output) == 0: - raise EnvironmentError('could not get commit comment for rev') - return stripped_output - d.addCallback(process) - return d + - def _get_commit_timestamp(self, rev): - # unix timestamp - args = ['log', rev, '--no-walk', r'--format=%ct'] - d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) - def process(git_output): - stripped_output = git_output.strip() - if self.usetimestamps: - try: - stamp = float(stripped_output) - except Exception, e: - log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % stripped_output) - raise e - return stamp - else: - return None - d.addCallback(process) - return d - - def _get_commit_files(self, rev): - args = ['log', rev, '--name-only', '--no-walk', r'--format=%n'] - d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) - def process(git_output): - fileList = git_output.split() - return fileList - d.addCallback(process) - return d - - def _get_commit_name(self, rev): - args = ['log', rev, '--no-walk', r'--format=%aE'] - d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) - def process(git_output): - stripped_output = git_output.strip() - if len(stripped_output) == 0: - raise EnvironmentError('could not get commit name for rev') - return stripped_output - d.addCallback(process) - return d - - @defer.defferedGenerator + @defer.deferredGenerator def _get_changes(self): + """update the changes if the hash doesnt match""" + log.msg('%s: polling repo at %s' % (self.name, self.repourl)) - self.lastPoll = time.time() # get the hash before updating @@ -189,11 +150,12 @@ @defer.deferredGenerator def _process_changes(self, unused_output): + # get the change list - revListArgs = ['log', 'HEAD..FETCH_HEAD', r'--format=%H'] self.changeCount = 0 - d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir, - env=dict(PATH=os.environ['PATH']), errortoo=False ) + if self.preHash == self.postHash: + return + d = self._change_list() wfd = defer.waitForDeferred(d) yield wfd results = wfd.getResult() @@ -202,21 +164,21 @@ revList = results.split() if not revList: return - revList.reverse() self.changeCount = len(revList) - log.msg('%s: processing %d changes: %s in "%s"' % (self.name, self.changeCount, revList, self.workdir) ) + # get metadata for changes and send them to master for rev in revList: + + # get metadata dl = defer.DeferredList([ self._get_commit_timestamp(rev), self._get_commit_name(rev), self._get_commit_files(rev), self._get_commit_comments(rev), ], consumeErrors=True) - wfd = defer.waitForDeferred(dl) yield wfd results = wfd.getResult() @@ -227,6 +189,7 @@ # just fail on the first error; they're probably all related! raise failures[0] + # send the change to the master timestamp, name, files, comments = [ r[1] for r in results ] d = self.master.addChange( who=name, @@ -248,16 +211,6 @@ # eat the failure to continue along the defered chain - we still want to catch up return None - def _catch_up(self, res): - if self.changeCount == 0: - log.msg('gitpoller: no changes, no catch_up') - return - log.msg('gitpoller: catching up to FETCH_HEAD') - args = ['reset', '--hard', 'FETCH_HEAD'] - d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH'])) - d.addCallback(self._convert_nonzero_to_failure) - return d - def _catch_up_failure(self, f): log.err(f) log.msg('%s: please resolve issues in local repo: %s' % (self.name, self.workdir)) @@ -277,7 +230,10 @@ d.addErrback(log.err, 'while stopping broken %s service' % self.name) return f + class HGPoller(Poller): + """poller for a mercurial source""" + def __init__(self, repourl, binary='hg', branch='default', **kwargs): Poller.__init__(self, repourl, binary=binary, branch=branch, **kwargs) @@ -295,35 +251,10 @@ # about the stderr or stdout from this command. We set errortoo=True to # avoid an errback from the deferred. The callback which will be added to this # deferred will not use the response. - d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=True ) + d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=True ) return d - @defer.deferredGenerator - def _process_changes(self, unused_ouput): - self.changeCount = 0 - if self.preHash == self.postHash: - return - range = '%s:%s' % (preHash, postHash) - d = utils.getProcessOutput(self.binary, ['log', '-r', range, '--template', '{node}\\n'], - path=self.workdir, - env=dict(PATH=os.environ['PATH']), errortoo=False ) - wfd = defer.waitForDeferred(d) - yield wfd - results = wfd.getResult() - - # process oldest change first - revList = results.split() - if not revList: - return - - revList.reverse() - self.changeCount = len(revList) - - log.msg('%s: processing %d changes: %s in "%s"' - % (self.name, self.changeCount, revList, self.workdir) ) - - for rev in revList: pass @defer.deferredGenerator def _hash(self) @@ -331,5 +262,79 @@ path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) return d + + def _change_list(self): + """ + return a deferred something-or-other that has the changes to be + processed. XXX the format is pretty particular + """ + range = '%s:%s' % (preHash, postHash) + d = utils.getProcessOutput(self.binary, ['log', '-r', range, '--template', '{node}\\n'], + path=self.workdir, + env=dict(PATH=os.environ['PATH']), errortoo=False ) + return d - + def _catch_up(self, res): + if self.changeCount == 0: + log.msg('%s: no changes, no catch_up' % self.name) + return + log.msg('%s: catching up' % self.name) + args = ['update'] + d = utils.getProcessOutputAndValue(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH'])) + d.addCallback(self._convert_nonzero_to_failure) + return d + + ### functions for retrieving various metadatas + + def _get_commit_timestamp(self, rev): + # unix timestamp + args = ['log', '-r', rev, '--template', '{date|hgdate}'] + d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) + def process(output): + stripped_output = output.strip() + if self.usetimestamps: + try: + _stamp, offset = output.split() + stamp = float(_stamp) + except Exception, e: + log.msg('%s: caught exception converting output "%s" to timestamp' % (self.name, stripped_output)) + raise e + return stamp + else: + return None + d.addCallback(process) + return d + + def _get_commit_name(self, rev): + """get the author of a commit""" + args = ['log', '-r', rev, '--template', '{author}'] + d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) + def process(output): + stripped_output = output.strip() + if len(stripped_output) == 0: + raise EnvironmentError('could not get commit name for rev') + return stripped_output + d.addCallback(process) + return d + + def _get_commit_files(self, rev): + """get the files associated with a commit""" + args = ['log', '-r', rev, '--template', '{files}'] + d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) + def process(output): + fileList = output.split() + return fileList + d.addCallback(process) + return d + + def _get_commit_comments(self, rev): + """get the commit message""" + args = ['log', '-r', rev, '--template', '{desc}'] + d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False ) + def process(output): + stripped_output = output.strip() + if len(stripped_output) == 0: + raise EnvironmentError('could not get commit comment for rev') + return stripped_output + d.addCallback(process) + return d