--- a/autobot/changes/poller.py	Fri Jan 14 08:06:00 2011 -0800
+++ b/autobot/changes/poller.py	Tue Jan 18 13:01:58 2011 -0800
@@ -13,9 +13,9 @@
 #
 # Copyright Buildbot Team Members
 
+import os
 import time
 import tempfile
-import os
 from twisted.python import log
 from twisted.internet import defer, utils
 
@@ -23,8 +23,10 @@
 from buildbot.changes import base
 
 class Poller(base.PollingChangeSource):
-    """This source will poll a remote git repo for changes and submit
-    them to the change master."""
+    """
+    This will poll a remote resource for changes and submit
+    them to the change master.
+    """
     
     compare_attrs = ["repourl", "branch", "workdir",
                      "pollInterval", "binary", "usetimestamps",
@@ -32,14 +34,15 @@
                      
     def __init__(self, repourl, binary=None, branch=None,
                  workdir=None, pollInterval=10*60, 
-                 binary='git', usetimestamps=True,
+                 binary=None, usetimestamps=True,
                  category=None, project=None,
                  pollinterval=-2):
+
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
+
         if project is None: project = ''
-
         self.repourl = repourl
         self.branch = branch
         self.pollInterval = pollInterval
@@ -54,12 +57,13 @@
         self.commitInfo  = {}
         self.initLock = defer.DeferredLock()
         
-        if self.workdir == None:
+        if not self.workdir:
             self.workdir = tempfile.gettempdir()
 
         self.name = self.__class__.__name__
 
     def startService(self):
+
         # initialize the repository we'll use to get changes; note that
         # startService is not an event-driven method, so this method will
         # instead acquire self.initLock immediately when it is called.
@@ -74,6 +78,9 @@
 
     @deferredLocked('initLock')
     def initRepository(self):
+        """initialize a repository or whatever"""
+
+        # make the directory, if necessary
         d = defer.succeed(None)
         def make_dir(_):
             dirpath = os.path.dirname(self.workdir.rstrip(os.sep))
@@ -82,16 +89,17 @@
                 os.makedirs(dirpath)
         d.addCallback(make_dir)
 
+        # perform the initialization
         def processCommand(command):
-            d = utils.getProcessOutputAndValue(self.gitbin,
+            d = utils.getProcessOutputAndValue(self.binary,
                     command, env=dict(PATH=os.environ['PATH']))
             d.addCallback(self._convert_nonzero_to_failure)
             d.addErrback(self._stop_on_failure)
-            return d            
-
+            return d
         for command in self.initializationCommands():
             d.addCallback(lambda _: processCommand(command[:]))
 
+        # finish up
         def log_finished(_):
             log.msg("%s: finished initializing working dir from %s" % (self.name, self.repourl)
         d.addCallback(log_finished)
@@ -113,60 +121,13 @@
         d.addCallback(self._catch_up)
         d.addErrback(self._catch_up_failure)
         return d
-
-    def _get_commit_comments(self, rev):
-        args = ['log', rev, '--no-walk', r'--format=%s%n%b']
-        d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        def process(git_output):
-            stripped_output = git_output.strip()
-            if len(stripped_output) == 0:
-                raise EnvironmentError('could not get commit comment for rev')
-            return stripped_output
-        d.addCallback(process)
-        return d
+            
 
-    def _get_commit_timestamp(self, rev):
-        # unix timestamp
-        args = ['log', rev, '--no-walk', r'--format=%ct']
-        d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        def process(git_output):
-            stripped_output = git_output.strip()
-            if self.usetimestamps:
-                try:
-                    stamp = float(stripped_output)
-                except Exception, e:
-                        log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % stripped_output)
-                        raise e
-                return stamp
-            else:
-                return None
-        d.addCallback(process)
-        return d
-
-    def _get_commit_files(self, rev):
-        args = ['log', rev, '--name-only', '--no-walk', r'--format=%n']
-        d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        def process(git_output):
-            fileList = git_output.split()
-            return fileList
-        d.addCallback(process)
-        return d
-            
-    def _get_commit_name(self, rev):
-        args = ['log', rev, '--no-walk', r'--format=%aE']
-        d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        def process(git_output):
-            stripped_output = git_output.strip()
-            if len(stripped_output) == 0:
-                raise EnvironmentError('could not get commit name for rev')
-            return stripped_output
-        d.addCallback(process)
-        return d
-
-    @defer.defferedGenerator
+    @defer.deferredGenerator
     def _get_changes(self):
+        """update the changes if the hash doesnt match"""
+        
         log.msg('%s: polling repo at %s' % (self.name, self.repourl))
-
         self.lastPoll = time.time()
 
         # get the hash before updating
@@ -189,11 +150,12 @@
 
     @defer.deferredGenerator
     def _process_changes(self, unused_output):
+
         # get the change list
-        revListArgs = ['log', 'HEAD..FETCH_HEAD', r'--format=%H']
         self.changeCount = 0
-        d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir,
-                                   env=dict(PATH=os.environ['PATH']), errortoo=False )
+        if self.preHash == self.postHash:
+            return
+        d = self._change_list()
         wfd = defer.waitForDeferred(d)
         yield wfd
         results = wfd.getResult()
@@ -202,21 +164,21 @@
         revList = results.split()
         if not revList:
             return
-
         revList.reverse()
         self.changeCount = len(revList)
-            
         log.msg('%s: processing %d changes: %s in "%s"'
                 % (self.name, self.changeCount, revList, self.workdir) )
 
+        # get metadata for changes and send them to master
         for rev in revList:
+
+            # get metadata
             dl = defer.DeferredList([
                 self._get_commit_timestamp(rev),
                 self._get_commit_name(rev),
                 self._get_commit_files(rev),
                 self._get_commit_comments(rev),
             ], consumeErrors=True)
-
             wfd = defer.waitForDeferred(dl)
             yield wfd
             results = wfd.getResult()
@@ -227,6 +189,7 @@
                 # just fail on the first error; they're probably all related!
                 raise failures[0]
 
+            # send the change to the master
             timestamp, name, files, comments = [ r[1] for r in results ]
             d = self.master.addChange(
                    who=name,
@@ -248,16 +211,6 @@
         # eat the failure to continue along the defered chain - we still want to catch up
         return None
         
-    def _catch_up(self, res):
-        if self.changeCount == 0:
-            log.msg('gitpoller: no changes, no catch_up')
-            return
-        log.msg('gitpoller: catching up to FETCH_HEAD')
-        args = ['reset', '--hard', 'FETCH_HEAD']
-        d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']))
-        d.addCallback(self._convert_nonzero_to_failure)
-        return d
-
     def _catch_up_failure(self, f):
         log.err(f)
         log.msg('%s: please resolve issues in local repo: %s' % (self.name, self.workdir))
@@ -277,7 +230,10 @@
             d.addErrback(log.err, 'while stopping broken %s service' % self.name)
         return f
 
+
 class HGPoller(Poller):
+    """poller for a mercurial source"""
+
     def __init__(self, repourl, binary='hg', branch='default', **kwargs):
         Poller.__init__(self, repourl, binary=binary, branch=branch, **kwargs)
 
@@ -295,35 +251,10 @@
         # about the stderr or stdout from this command. We set errortoo=True to
         # avoid an errback from the deferred. The callback which will be added to this
         # deferred will not use the response.
-        d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=True )
+        d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=True )
 
         return d
 
-    @defer.deferredGenerator
-    def _process_changes(self, unused_ouput):
-        self.changeCount = 0
-        if self.preHash == self.postHash:
-            return
-        range = '%s:%s' % (preHash, postHash)
-        d = utils.getProcessOutput(self.binary, ['log', '-r', range, '--template', '{node}\\n'],
-                                   path=self.workdir,
-                                   env=dict(PATH=os.environ['PATH']), errortoo=False )
-        wfd = defer.waitForDeferred(d)
-        yield wfd
-        results = wfd.getResult()
-
-        # process oldest change first
-        revList = results.split()
-        if not revList:
-            return
-
-        revList.reverse()
-        self.changeCount = len(revList)
-
-        log.msg('%s: processing %d changes: %s in "%s"'
-                % (self.name, self.changeCount, revList, self.workdir) )
-
-        for rev in revList: pass
 
     @defer.deferredGenerator
     def _hash(self)
@@ -331,5 +262,79 @@
                                    path=self.workdir,
                                    env=dict(PATH=os.environ['PATH']), errortoo=False )
         return d
+
+    def _change_list(self):
+        """
+        return a deferred something-or-other that has the changes to be
+        processed.  XXX the format is pretty particular
+        """
+        range = '%s:%s' % (preHash, postHash)
+        d = utils.getProcessOutput(self.binary, ['log', '-r', range, '--template', '{node}\\n'],
+                                   path=self.workdir,
+                                   env=dict(PATH=os.environ['PATH']), errortoo=False )
+        return d
         
-        
+    def _catch_up(self, res):
+        if self.changeCount == 0:
+            log.msg('%s: no changes, no catch_up' % self.name)
+            return
+        log.msg('%s: catching up' % self.name)
+        args = ['update']
+        d = utils.getProcessOutputAndValue(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']))
+        d.addCallback(self._convert_nonzero_to_failure)
+        return d
+
+    ### functions for retrieving various metadatas
+
+    def _get_commit_timestamp(self, rev):
+        # unix timestamp
+        args = ['log', '-r', rev, '--template', '{date|hgdate}']
+        d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+        def process(output):
+            stripped_output = output.strip()
+            if self.usetimestamps:
+                try:
+                    _stamp, offset = output.split()
+                    stamp = float(_stamp)
+                except Exception, e:
+                        log.msg('%s: caught exception converting output "%s" to timestamp' % (self.name, stripped_output))
+                        raise e
+                return stamp
+            else:
+                return None
+        d.addCallback(process)
+        return d
+
+     def _get_commit_name(self, rev):
+         """get the author of a commit"""
+         args = ['log', '-r', rev, '--template', '{author}']
+         d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+         def process(output):
+             stripped_output = output.strip()
+             if len(stripped_output) == 0:
+                 raise EnvironmentError('could not get commit name for rev')
+             return stripped_output
+         d.addCallback(process)
+         return d
+
+     def _get_commit_files(self, rev):
+         """get the files associated with a commit"""
+         args = ['log', '-r', rev, '--template', '{files}']
+         d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+         def process(output):
+             fileList = output.split()
+             return fileList
+         d.addCallback(process)
+         return d
+
+     def _get_commit_comments(self, rev):
+         """get the commit message"""
+         args = ['log', '-r', rev, '--template', '{desc}']
+         d = utils.getProcessOutput(self.binary, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
+         def process(output):
+             stripped_output = output.strip()
+             if len(stripped_output) == 0:
+                 raise EnvironmentError('could not get commit comment for rev')
+             return stripped_output
+         d.addCallback(process)
+         return d