--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/INSTALL.sh	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,27 @@
+#!/usr/bin/bash
+
+# installation script for fetch
+# fetch stuff from the interwebs
+
+REPO='http://k0s.org/mozilla/hg/fetch'
+DEST='fetch' # name of the virtualenv
+
+if [ "$#" != "0" ]
+then
+    DEST=$1
+fi
+
+if which virtualenv
+then
+    virtualenv ${DEST}
+else
+    curl https://bitbucket.org/ianb/virtualenv/raw/tip/virtualenv.py | python - ${DEST}
+fi
+cd ${DEST}
+. bin/activate # linux only
+mkdir src
+cd src
+hg clone ${REPO}
+cd fetch
+python setup.py develop
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/README.txt	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,17 @@
+fetch
+===========
+
+fetch stuff from the interwebs
+
+Format
+------
+
+[URL] [Destination] [Type]
+
+
+----
+
+Jeff Hammel
+
+
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/example.txt	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,3 @@
+# example manifest for fetch
+
+http://k0s.org/geekcode . file

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/fetch/__init__.py	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,1 @@
+#

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/fetch/main.py	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+
+"""
+fetch stuff from the interwebs
+"""
+
+import os
+import sys
+import optparse
+
+class Fetcher(object):
+  """abstract base class for resource fetchers"""
+
+  @classmethod
+  def match(cls, _type):
+    return _type == cls.type
+
+  def __init__(self, url):
+    self.url = url
+
+  def __call__(self, dest):
+    raise NotImplementedError
+
+
+import urllib2
+
+class FileFetcher(object):
+  """fetch a single file"""
+  
+  type = 'file'
+
+  @classmethod
+  def download(cls, url):
+    return urllib2.urlopen(url).read()
+
+  def __call__(self, dest):
+    if os.path.isdir(dest):
+      filename = url.rsplit('/', 1)[-1]
+      dest = os.path.join(dest, filename)
+    f = file(dest, 'w')
+    f.write(self.download(self.url))
+    f.close()
+
+class TarballFetcher(object):
+  """fetch and extract a tarball"""
+
+  type = 'tar'
+
+class HgFetcher(object):
+  """checkout a mercurial repository"""
+
+class GitFetcher(object):
+  """checkout a git repository"""
+
+fetchers = [FileFetcher]
+
+class Fetch(object):
+  
+  def __init__(self, fetchers, relative_to=None, strict=True):
+    self.fetchers = fetchers
+    self.relative_to = relative_to
+    self.strict = strict
+
+  def fetcher(self, _type):
+    """find the fetcher for the appropriate type"""
+    for fetcher in fetchers:
+      if fetcher.match(_type):
+        return fetcher
+
+  def __call__(self, url, destination, type, **options):
+    fetcher = self.fetcher(type)
+    assert fetcher is not None
+    fetcher = fetcher(url, **options)
+    fetcher(destination)
+
+  def fetch(self, *items):
+    for item in items:
+      self(item['url'])
+
+format_string = "[URL] [destination] [type] <options>"
+def read_manifests(*manifests):
+  """
+  read some manifests and return the items
+
+  Format:
+  %s
+  """ % format_string
+
+  # sanity check
+  assert not [i for i in manifest if not.os.path.exists(i)]
+
+  retval = []
+
+  for manifest in manifests:
+    for line in file(i).readlines():
+      line = line.strip()
+      if line.startswith('#'):
+        pass
+      line = line.split()
+      if len(line) not in (3,4):
+        raise Exception("Format should be: %s; line %s" % (format_string, line))
+      options = {}
+      if len(line) == 4:
+        option_string = line.pop().rstrip(',')
+        try:
+          options = dict([[j.strip() for j in i.split('=', 1)]
+                          for i in option_string.split(',')])
+        except:
+          raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string)
+
+      url, dest, _type = line
+      retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest))
+  return retval
+
+def main(args=sys.argv[:]):
+
+  # parse command line options
+  usage = '%prog [options] manifest [manifest] [...]'
+
+  # description formatter
+  class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
+    def format_description(self, description):
+      if description:
+        return description + '\n'
+      else:
+        return ''
+  
+  parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter())
+  parser.add_option('-o', '--output',
+                    help="output relative to this location vs. the manifest location")
+  parser.add_option('-d', '--dest',
+                    action='append',
+                    help="output only these destinations")
+  parser.add_option('-s', '--strict',
+                    action='store_true', default=False,
+                    help="fail on error")
+  options, args = parser.parse_args(args)
+
+  if not args:
+    parser.print_help()
+    parser.exit()
+
+  items = read_manifests(*args)
+  fetch = Fetch(fetchers, strict=options.strict)
+
+  # ensure the all the required fetchers are available
+  # TODO: to Fetch class
+  types = set([i['type'] for i in items])
+  assert not [i for i in types
+              if [True for fetcher in fetchers if fetcher.match(i)]]
+
+  # download the files
+  fetch.fetch(*items)
+
+if __name__ == '__main__':
+  main()  
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/fetch/web.py	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+"""
+web handler for fetch
+"""
+
+from webob import Request, Response, exc
+
+class Handler(object):
+
+    def __init__(self, **kw):
+        pass
+
+    def __call__(self, environ, start_response):
+        request = Request(environ)
+        response = Response(content_type='text/plain',
+                            body="fetch")
+        return response(environ, start_response)
+
+if __name__ == '__main__':
+    from wsgiref import simple_server
+    app = Handler()
+    server = simple_server.make_server(host='0.0.0.0', port=8080, app=app)
+    server.serve_forever()
+          
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/setup.py	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,35 @@
+import os
+from setuptools import setup, find_packages
+
+try:
+    here = os.path.dirname(os.path.abspath(__file__))
+    description = file(os.path.join(here, 'README.txt')).read()
+except IOError: 
+    description = ''
+
+version = "0.0"
+
+dependencies = ['MakeItSo', 'webob']
+
+setup(name='fetch',
+      version=version,
+      description="fetch stuff from the interwebs",
+      long_description=description,
+      classifiers=[], # Get strings from http://www.python.org/pypi?%3Aaction=list_classifiers
+      author='Jeff Hammel',
+      author_email='jhammel@mozilla.com',
+      url='http://k0s.org/mozilla/fetch',
+      license='MPL',
+      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=dependencies,
+      entry_points="""
+      # -*- Entry points: -*-
+
+      [console_scripts]
+      fetch = fetch.main:main
+      """,
+      )
+      
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/doctest.txt	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,11 @@
+Test fetch
+================
+
+The obligatory imports:
+
+    >>> import fetch
+
+Run some tests.  This test will fail, please fix it:
+
+    >>> assert True == False
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test.py	Sun Sep 18 10:33:04 2011 -0700
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+"""
+doctest runner
+"""
+
+import doctest
+import os
+import sys
+from optparse import OptionParser
+
+
+def run_tests(raise_on_error=False, report_first=False):
+
+    # add results here
+    results = {}
+
+    # doctest arguments
+    directory = os.path.dirname(os.path.abspath(__file__))
+    extraglobs = {'here': directory}
+    doctest_args = dict(extraglobs=extraglobs, raise_on_error=raise_on_error)
+    if report_first:
+        doctest_args['optionflags'] = doctest.REPORT_ONLY_FIRST_FAILURE
+
+    # gather tests
+    tests =  [ test for test in os.listdir(directory)
+               if test.endswith('.txt') ]
+
+    # run the tests
+    for test in tests:
+        try:
+            results[test] = doctest.testfile(test, **doctest_args)
+        except doctest.DocTestFailure, failure:
+            raise
+        except doctest.UnexpectedException, failure:
+            raise failure.exc_info[0], failure.exc_info[1], failure.exc_info[2]
+
+    return results
+
+def main(args=sys.argv[1:]):
+
+    # parse command line args
+    parser = OptionParser(description=__doc__)
+    parser.add_option('--raise', dest='raise_on_error',
+                      default=False, action='store_true',
+                      help="raise on first error")
+    parser.add_option('--report-first', dest='report_first',
+                      default=False, action='store_true',
+                      help="report the first error only (all tests will still run)")
+    options, args = parser.parse_args(args)
+
+    # run the tests
+    results = run_tests(**options.__dict__)
+    if sum([i.failed for i in results.values()]):
+        sys.exit(1) # error
+                
+
+if __name__ == '__main__':
+    main()
+