Mercurial > hg > fetch

changeset 15:bc7d6763357e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
clean up spacing
author Jeff Hammel <jhammel@mozilla.com>
date Wed, 09 Nov 2011 16:29:58 -0800
parents 626b027134ea
children c77d29a10e08
files fetch.py
diffstat 1 files changed, 135 insertions(+), 142 deletions(-) [+]
line wrap: on
line diff
--- a/fetch.py	Wed Nov 09 16:20:30 2011 -0800
+++ b/fetch.py	Wed Nov 09 16:29:58 2011 -0800
@@ -11,26 +11,24 @@
 __all__ = ['Fetcher', 'Fetch', 'main']
 
 def which(executable, path=os.environ['PATH']):
-  """python equivalent of which; should really be in the stdlib"""
-  # XXX from https://github.com/mozautomation/mozmill/blob/master/mozrunner/mozrunner/utils.py
-  dirs = path.split(os.pathsep)
-  for dir in dirs:
-    if os.path.isfile(os.path.join(dir, executable)):
-      return os.path.join(dir, executable)
-
+    """python equivalent of which; should really be in the stdlib"""
+    dirs = path.split(os.pathsep)
+    for dir in dirs:
+        if os.path.isfile(os.path.join(dir, executable)):
+            return os.path.join(dir, executable)
 
 class Fetcher(object):
-  """abstract base class for resource fetchers"""
+    """abstract base class for resource fetchers"""
 
-  @classmethod
-  def match(cls, _type):
-    return _type == cls.type
+    @classmethod
+    def match(cls, _type):
+        return _type == cls.type
 
-  def __init__(self, url):
-    self.url = url
+    def __init__(self, url):
+        self.url = url
 
-  def __call__(self, dest):
-    raise NotImplementedError
+    def __call__(self, dest):
+        raise NotImplementedError
 
 ### standard dispatchers - always available
 
@@ -39,35 +37,35 @@
 from StringIO import StringIO
 
 class FileFetcher(Fetcher):
-  """fetch a single file"""
+    """fetch a single file"""
   
-  type = 'file'
+    type = 'file'
 
-  @classmethod
-  def download(cls, url):
-    return urllib2.urlopen(url).read()
+    @classmethod
+    def download(cls, url):
+        return urllib2.urlopen(url).read()
 
-  def __call__(self, dest):
-    if os.path.isdir(dest):
-      filename = self.url.rsplit('/', 1)[-1]
-      dest = os.path.join(dest, filename)
-    f = file(dest, 'w')
-    f.write(self.download(self.url))
-    f.close()
+    def __call__(self, dest):
+        if os.path.isdir(dest):
+            filename = self.url.rsplit('/', 1)[-1]
+            dest = os.path.join(dest, filename)
+        f = file(dest, 'w')
+        f.write(self.download(self.url))
+        f.close()
 
 
 class TarballFetcher(FileFetcher):
-  """fetch and extract a tarball"""
+    """fetch and extract a tarball"""
 
-  type = 'tar'
+    type = 'tar'
 
-  def __call__(self, dest):
-    assert os.path.isdir(dest)
-    buffer = StringIO()
-    buffer.write(self.download(self.url))
-    buffer.seek(0)
-    tf = tarfile.open(mode='r', fileobj=buffer)
-    tf.extract(dest)
+    def __call__(self, dest):
+        assert os.path.isdir(dest)
+        buffer = StringIO()
+        buffer.write(self.download(self.url))
+        buffer.seek(0)
+        tf = tarfile.open(mode='r', fileobj=buffer)
+        tf.extract(dest)
 
 fetchers = [FileFetcher, TarballFetcher]
 
@@ -77,144 +75,139 @@
 
 if which('hg'):
 
-  class HgFetcher(Fetcher):
-    """checkout a mercurial repository"""
-    type = 'hg'
+    class HgFetcher(Fetcher):
+        """checkout a mercurial repository"""
+        type = 'hg'
 
-  def __call__(self, dest):
-    if os.path.exits(dest):
-      assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg'))
-      pass # TODO
+        def __call__(self, dest):
+            if os.path.exits(dest):
+                assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg'))
+            pass # TODO
 
-  fetchers.append(HgFetcher)
+    fetchers.append(HgFetcher)
 
-class GitFetcher(Fetcher):
-  """checkout a git repository"""
-  type = 'git'
-
+if which('git'):
+    class GitFetcher(Fetcher):
+        """checkout a git repository"""
+        type = 'git'
 
 fetchers = dict([(i.__name__, i) for i in fetchers])
 __all__ += fetchers.keys()
 
-
 class Fetch(object):
   
-  def __init__(self, fetchers, relative_to=None, strict=True):
-    self.fetchers = fetchers
-    self.relative_to = relative_to
-    self.strict = strict
+    def __init__(self, fetchers, relative_to=None, strict=True):
+        self.fetchers = fetchers
+        self.relative_to = relative_to
+        self.strict = strict
 
-  def fetcher(self, _type):
-    """find the fetcher for the appropriate type"""
-    for fetcher in fetchers:
-      if fetcher.match(_type):
-        return fetcher
+    def fetcher(self, _type):
+        """find the fetcher for the appropriate type"""
+        for fetcher in fetchers:
+            if fetcher.match(_type):
+                return fetcher
 
-  def __call__(self, url, destination, type, **options):
-    fetcher = self.fetcher(type)
-    assert fetcher is not None
-    fetcher = fetcher(url, **options)
-    fetcher(destination)
-
-  def fetch(self, *items):
+    def __call__(self, url, destination, type, **options):
+        fetcher = self.fetcher(type)
+        assert fetcher is not None, "No fetcher found for type '%s'" % type
+        fetcher = fetcher(url, **options)
+        fetcher(destination)
 
-    if self.strict:
-      # ensure all the required fetchers are available
-      types = set([i['type'] for i in items])
-      assert not [i for i in types
-                  if [True for fetcher in fetchers if fetcher.match(i)]]
+    def fetch(self, *items):
 
-    for item in items:
+        if self.strict:
+            # ensure all the required fetchers are available
+            types = set([i['type'] for i in items])
+            assert not [i for i in types
+                        if [True for fetcher in fetchers if fetcher.match(i)]]
 
-      # fix up relative paths
-      dest = item['dest']
-      if not os.path.isabs(dest):
-        if self.relative_to:
-          dest = os.path.join(self.relative_to, dest)
-        else:
-          dest = os.path.join(os.path.dirname(os.path.abspath(item['manifest'])), dest)
+        for item in items:
 
-      # fetch the items
-      self(item['url'], destination=dest, type=item['type'], **item['options'])
+            # fix up relative paths
+            dest = item['dest']
+            if not os.path.isabs(dest):
+                relative_to = self.relative_to or os.path.dirname(os.path.abspath(item['manifest']))
+                dest = os.path.join(relative_to, dest)
 
+            # fetch the items
+            self(item['url'], destination=dest, type=item['type'], **item['options'])
 
 format_string = "[URL] [destination] [type] <options>"
 def read_manifests(*manifests):
-  """
-  read some manifests and return the items
+    """
+    read some manifests and return the items
+    
+    Format:
+    %s
+    """ % format_string
 
-  Format:
-  %s
-  """ % format_string
+    # sanity check
+    assert not [i for i in manifests if not os.path.exists(i)]
 
-  # sanity check
-  assert not [i for i in manifests if not os.path.exists(i)]
-
-  retval = []
+    retval = []
 
-  for manifest in manifests:
-    for line in file(i).readlines():
-      line = line.strip()
-      if line.startswith('#') or not line:
-        continue
-      line = line.split()
-      if len(line) not in (3,4):
-        raise Exception("Format should be: %s; line %s" % (format_string, line))
-      options = {}
-      if len(line) == 4:
-        option_string = line.pop().rstrip(',')
-        try:
-          options = dict([[j.strip() for j in i.split('=', 1)]
-                          for i in option_string.split(',')])
-        except:
-          raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string)
+    for manifest in manifests:
+        for line in file(i).readlines():
+            line = line.strip()
+            if line.startswith('#') or not line:
+                continue
+            line = line.split()
+            if len(line) not in (3,4):
+                raise Exception("Format should be: %s; line %s" % (format_string, line))
+            options = {}
+            if len(line) == 4:
+                option_string = line.pop().rstrip(',')
+                try:
+                    options = dict([[j.strip() for j in i.split('=', 1)]
+                                    for i in option_string.split(',')])
+                except:
+                    raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string)
 
-      url, dest, _type = line
-      retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest))
-  return retval
+            url, dest, _type = line
+            retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest))
+    return retval
 
 def main(args=sys.argv[1:]):
 
-  # parse command line options
-  usage = '%prog [options] manifest [manifest] [...]'
+    # parse command line options
+    usage = '%prog [options] manifest [manifest] [...]'
 
-  # description formatter
-  class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
-    def format_description(self, description):
-      if description:
-        return description + '\n'
-      else:
-        return ''
+    class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
+        def format_description(self, description):
+            if description:
+                return description + '\n'
+            else:
+                return ''
   
-  parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter())
-  parser.add_option('-o', '--output',
-                    help="output relative to this location vs. the manifest location")
-  parser.add_option('-d', '--dest',
-                    action='append',
-                    help="output only these destinations")
-  parser.add_option('-s', '--strict',
-                    action='store_true', default=False,
-                    help="fail on error")
-  parser.add_option('--list-fetchers', dest='list_fetchers',
-                    action='store_true', default=False,
-                    help='list available fetchers and exit')
-  options, args = parser.parse_args(args)
+    parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter())
+    parser.add_option('-o', '--output',
+                      help="output relative to this location vs. the manifest location")
+    parser.add_option('-d', '--dest',
+                      action='append',
+                      help="output only these destinations")
+    parser.add_option('-s', '--strict',
+                      action='store_true', default=False,
+                      help="fail on error")
+    parser.add_option('--list-fetchers', dest='list_fetchers',
+                      action='store_true', default=False,
+                      help='list available fetchers and exit')
+    options, args = parser.parse_args(args)
 
-  if options.list_fetchers:
-    for name in sorted(fetchers.keys()):
-      print name
-    parser.exit()
+    if options.list_fetchers:
+        for name in sorted(fetchers.keys()):
+            print name
+        parser.exit()
 
-  if not args:
-    parser.print_help()
-    parser.exit()
+    if not args:
+        parser.print_help()
+        parser.exit()
 
-  items = read_manifests(*args)
-  fetch = Fetch(fetchers.values(), strict=options.strict)
+    items = read_manifests(*args)
+    fetch = Fetch(fetchers.values(), strict=options.strict)
 
-  # download the files
-  fetch.fetch(*items)
+    # download the files
+    fetch.fetch(*items)
 
 if __name__ == '__main__':
-  main()  
+    main()