# HG changeset patch # User Jeff Hammel # Date 1316367184 25200 # Node ID 3497a30190d26b9efd74f04581ab89096ebd9d89 initial commit of fetch, WIP diff -r 000000000000 -r 3497a30190d2 INSTALL.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/INSTALL.sh Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,27 @@ +#!/usr/bin/bash + +# installation script for fetch +# fetch stuff from the interwebs + +REPO='http://k0s.org/mozilla/hg/fetch' +DEST='fetch' # name of the virtualenv + +if [ "$#" != "0" ] +then + DEST=$1 +fi + +if which virtualenv +then + virtualenv ${DEST} +else + curl https://bitbucket.org/ianb/virtualenv/raw/tip/virtualenv.py | python - ${DEST} +fi +cd ${DEST} +. bin/activate # linux only +mkdir src +cd src +hg clone ${REPO} +cd fetch +python setup.py develop + diff -r 000000000000 -r 3497a30190d2 README.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.txt Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,17 @@ +fetch +=========== + +fetch stuff from the interwebs + +Format +------ + +[URL] [Destination] [Type] + + +---- + +Jeff Hammel + + + diff -r 000000000000 -r 3497a30190d2 example.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/example.txt Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,3 @@ +# example manifest for fetch + +http://k0s.org/geekcode . file diff -r 000000000000 -r 3497a30190d2 fetch/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/fetch/__init__.py Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,1 @@ +# diff -r 000000000000 -r 3497a30190d2 fetch/main.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/fetch/main.py Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,157 @@ +#!/usr/bin/env python + +""" +fetch stuff from the interwebs +""" + +import os +import sys +import optparse + +class Fetcher(object): + """abstract base class for resource fetchers""" + + @classmethod + def match(cls, _type): + return _type == cls.type + + def __init__(self, url): + self.url = url + + def __call__(self, dest): + raise NotImplementedError + + +import urllib2 + +class FileFetcher(object): + """fetch a single file""" + + type = 'file' + + @classmethod + def download(cls, url): + return urllib2.urlopen(url).read() + + def __call__(self, dest): + if os.path.isdir(dest): + filename = url.rsplit('/', 1)[-1] + dest = os.path.join(dest, filename) + f = file(dest, 'w') + f.write(self.download(self.url)) + f.close() + +class TarballFetcher(object): + """fetch and extract a tarball""" + + type = 'tar' + +class HgFetcher(object): + """checkout a mercurial repository""" + +class GitFetcher(object): + """checkout a git repository""" + +fetchers = [FileFetcher] + +class Fetch(object): + + def __init__(self, fetchers, relative_to=None, strict=True): + self.fetchers = fetchers + self.relative_to = relative_to + self.strict = strict + + def fetcher(self, _type): + """find the fetcher for the appropriate type""" + for fetcher in fetchers: + if fetcher.match(_type): + return fetcher + + def __call__(self, url, destination, type, **options): + fetcher = self.fetcher(type) + assert fetcher is not None + fetcher = fetcher(url, **options) + fetcher(destination) + + def fetch(self, *items): + for item in items: + self(item['url']) + +format_string = "[URL] [destination] [type] " +def read_manifests(*manifests): + """ + read some manifests and return the items + + Format: + %s + """ % format_string + + # sanity check + assert not [i for i in manifest if not.os.path.exists(i)] + + retval = [] + + for manifest in manifests: + for line in file(i).readlines(): + line = line.strip() + if line.startswith('#'): + pass + line = line.split() + if len(line) not in (3,4): + raise Exception("Format should be: %s; line %s" % (format_string, line)) + options = {} + if len(line) == 4: + option_string = line.pop().rstrip(',') + try: + options = dict([[j.strip() for j in i.split('=', 1)] + for i in option_string.split(',')]) + except: + raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) + + url, dest, _type = line + retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) + return retval + +def main(args=sys.argv[:]): + + # parse command line options + usage = '%prog [options] manifest [manifest] [...]' + + # description formatter + class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): + def format_description(self, description): + if description: + return description + '\n' + else: + return '' + + parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) + parser.add_option('-o', '--output', + help="output relative to this location vs. the manifest location") + parser.add_option('-d', '--dest', + action='append', + help="output only these destinations") + parser.add_option('-s', '--strict', + action='store_true', default=False, + help="fail on error") + options, args = parser.parse_args(args) + + if not args: + parser.print_help() + parser.exit() + + items = read_manifests(*args) + fetch = Fetch(fetchers, strict=options.strict) + + # ensure the all the required fetchers are available + # TODO: to Fetch class + types = set([i['type'] for i in items]) + assert not [i for i in types + if [True for fetcher in fetchers if fetcher.match(i)]] + + # download the files + fetch.fetch(*items) + +if __name__ == '__main__': + main() + diff -r 000000000000 -r 3497a30190d2 fetch/web.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/fetch/web.py Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +""" +web handler for fetch +""" + +from webob import Request, Response, exc + +class Handler(object): + + def __init__(self, **kw): + pass + + def __call__(self, environ, start_response): + request = Request(environ) + response = Response(content_type='text/plain', + body="fetch") + return response(environ, start_response) + +if __name__ == '__main__': + from wsgiref import simple_server + app = Handler() + server = simple_server.make_server(host='0.0.0.0', port=8080, app=app) + server.serve_forever() + + diff -r 000000000000 -r 3497a30190d2 setup.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.py Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,35 @@ +import os +from setuptools import setup, find_packages + +try: + here = os.path.dirname(os.path.abspath(__file__)) + description = file(os.path.join(here, 'README.txt')).read() +except IOError: + description = '' + +version = "0.0" + +dependencies = ['MakeItSo', 'webob'] + +setup(name='fetch', + version=version, + description="fetch stuff from the interwebs", + long_description=description, + classifiers=[], # Get strings from http://www.python.org/pypi?%3Aaction=list_classifiers + author='Jeff Hammel', + author_email='jhammel@mozilla.com', + url='http://k0s.org/mozilla/fetch', + license='MPL', + packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + include_package_data=True, + zip_safe=False, + install_requires=dependencies, + entry_points=""" + # -*- Entry points: -*- + + [console_scripts] + fetch = fetch.main:main + """, + ) + + diff -r 000000000000 -r 3497a30190d2 tests/doctest.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/doctest.txt Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,11 @@ +Test fetch +================ + +The obligatory imports: + + >>> import fetch + +Run some tests. This test will fail, please fix it: + + >>> assert True == False + diff -r 000000000000 -r 3497a30190d2 tests/test.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test.py Sun Sep 18 10:33:04 2011 -0700 @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +""" +doctest runner +""" + +import doctest +import os +import sys +from optparse import OptionParser + + +def run_tests(raise_on_error=False, report_first=False): + + # add results here + results = {} + + # doctest arguments + directory = os.path.dirname(os.path.abspath(__file__)) + extraglobs = {'here': directory} + doctest_args = dict(extraglobs=extraglobs, raise_on_error=raise_on_error) + if report_first: + doctest_args['optionflags'] = doctest.REPORT_ONLY_FIRST_FAILURE + + # gather tests + tests = [ test for test in os.listdir(directory) + if test.endswith('.txt') ] + + # run the tests + for test in tests: + try: + results[test] = doctest.testfile(test, **doctest_args) + except doctest.DocTestFailure, failure: + raise + except doctest.UnexpectedException, failure: + raise failure.exc_info[0], failure.exc_info[1], failure.exc_info[2] + + return results + +def main(args=sys.argv[1:]): + + # parse command line args + parser = OptionParser(description=__doc__) + parser.add_option('--raise', dest='raise_on_error', + default=False, action='store_true', + help="raise on first error") + parser.add_option('--report-first', dest='report_first', + default=False, action='store_true', + help="report the first error only (all tests will still run)") + options, args = parser.parse_args(args) + + # run the tests + results = run_tests(**options.__dict__) + if sum([i.failed for i in results.values()]): + sys.exit(1) # error + + +if __name__ == '__main__': + main() +