# HG changeset patch # User Jeff Hammel # Date 1320884998 28800 # Node ID bc7d6763357e95d94a6d0cc2f5f93ee67e0420d8 # Parent 626b027134ea5e1e8719d4136ce0be21e83489ad clean up spacing diff -r 626b027134ea -r bc7d6763357e fetch.py --- a/fetch.py Wed Nov 09 16:20:30 2011 -0800 +++ b/fetch.py Wed Nov 09 16:29:58 2011 -0800 @@ -11,26 +11,24 @@ __all__ = ['Fetcher', 'Fetch', 'main'] def which(executable, path=os.environ['PATH']): - """python equivalent of which; should really be in the stdlib""" - # XXX from https://github.com/mozautomation/mozmill/blob/master/mozrunner/mozrunner/utils.py - dirs = path.split(os.pathsep) - for dir in dirs: - if os.path.isfile(os.path.join(dir, executable)): - return os.path.join(dir, executable) - + """python equivalent of which; should really be in the stdlib""" + dirs = path.split(os.pathsep) + for dir in dirs: + if os.path.isfile(os.path.join(dir, executable)): + return os.path.join(dir, executable) class Fetcher(object): - """abstract base class for resource fetchers""" + """abstract base class for resource fetchers""" - @classmethod - def match(cls, _type): - return _type == cls.type + @classmethod + def match(cls, _type): + return _type == cls.type - def __init__(self, url): - self.url = url + def __init__(self, url): + self.url = url - def __call__(self, dest): - raise NotImplementedError + def __call__(self, dest): + raise NotImplementedError ### standard dispatchers - always available @@ -39,35 +37,35 @@ from StringIO import StringIO class FileFetcher(Fetcher): - """fetch a single file""" + """fetch a single file""" - type = 'file' + type = 'file' - @classmethod - def download(cls, url): - return urllib2.urlopen(url).read() + @classmethod + def download(cls, url): + return urllib2.urlopen(url).read() - def __call__(self, dest): - if os.path.isdir(dest): - filename = self.url.rsplit('/', 1)[-1] - dest = os.path.join(dest, filename) - f = file(dest, 'w') - f.write(self.download(self.url)) - f.close() + def __call__(self, dest): + if os.path.isdir(dest): + filename = self.url.rsplit('/', 1)[-1] + dest = os.path.join(dest, filename) + f = file(dest, 'w') + f.write(self.download(self.url)) + f.close() class TarballFetcher(FileFetcher): - """fetch and extract a tarball""" + """fetch and extract a tarball""" - type = 'tar' + type = 'tar' - def __call__(self, dest): - assert os.path.isdir(dest) - buffer = StringIO() - buffer.write(self.download(self.url)) - buffer.seek(0) - tf = tarfile.open(mode='r', fileobj=buffer) - tf.extract(dest) + def __call__(self, dest): + assert os.path.isdir(dest) + buffer = StringIO() + buffer.write(self.download(self.url)) + buffer.seek(0) + tf = tarfile.open(mode='r', fileobj=buffer) + tf.extract(dest) fetchers = [FileFetcher, TarballFetcher] @@ -77,144 +75,139 @@ if which('hg'): - class HgFetcher(Fetcher): - """checkout a mercurial repository""" - type = 'hg' + class HgFetcher(Fetcher): + """checkout a mercurial repository""" + type = 'hg' - def __call__(self, dest): - if os.path.exits(dest): - assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg')) - pass # TODO + def __call__(self, dest): + if os.path.exits(dest): + assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg')) + pass # TODO - fetchers.append(HgFetcher) + fetchers.append(HgFetcher) -class GitFetcher(Fetcher): - """checkout a git repository""" - type = 'git' - +if which('git'): + class GitFetcher(Fetcher): + """checkout a git repository""" + type = 'git' fetchers = dict([(i.__name__, i) for i in fetchers]) __all__ += fetchers.keys() - class Fetch(object): - def __init__(self, fetchers, relative_to=None, strict=True): - self.fetchers = fetchers - self.relative_to = relative_to - self.strict = strict + def __init__(self, fetchers, relative_to=None, strict=True): + self.fetchers = fetchers + self.relative_to = relative_to + self.strict = strict - def fetcher(self, _type): - """find the fetcher for the appropriate type""" - for fetcher in fetchers: - if fetcher.match(_type): - return fetcher + def fetcher(self, _type): + """find the fetcher for the appropriate type""" + for fetcher in fetchers: + if fetcher.match(_type): + return fetcher - def __call__(self, url, destination, type, **options): - fetcher = self.fetcher(type) - assert fetcher is not None - fetcher = fetcher(url, **options) - fetcher(destination) - - def fetch(self, *items): + def __call__(self, url, destination, type, **options): + fetcher = self.fetcher(type) + assert fetcher is not None, "No fetcher found for type '%s'" % type + fetcher = fetcher(url, **options) + fetcher(destination) - if self.strict: - # ensure all the required fetchers are available - types = set([i['type'] for i in items]) - assert not [i for i in types - if [True for fetcher in fetchers if fetcher.match(i)]] + def fetch(self, *items): - for item in items: + if self.strict: + # ensure all the required fetchers are available + types = set([i['type'] for i in items]) + assert not [i for i in types + if [True for fetcher in fetchers if fetcher.match(i)]] - # fix up relative paths - dest = item['dest'] - if not os.path.isabs(dest): - if self.relative_to: - dest = os.path.join(self.relative_to, dest) - else: - dest = os.path.join(os.path.dirname(os.path.abspath(item['manifest'])), dest) + for item in items: - # fetch the items - self(item['url'], destination=dest, type=item['type'], **item['options']) + # fix up relative paths + dest = item['dest'] + if not os.path.isabs(dest): + relative_to = self.relative_to or os.path.dirname(os.path.abspath(item['manifest'])) + dest = os.path.join(relative_to, dest) + # fetch the items + self(item['url'], destination=dest, type=item['type'], **item['options']) format_string = "[URL] [destination] [type] " def read_manifests(*manifests): - """ - read some manifests and return the items + """ + read some manifests and return the items + + Format: + %s + """ % format_string - Format: - %s - """ % format_string + # sanity check + assert not [i for i in manifests if not os.path.exists(i)] - # sanity check - assert not [i for i in manifests if not os.path.exists(i)] - - retval = [] + retval = [] - for manifest in manifests: - for line in file(i).readlines(): - line = line.strip() - if line.startswith('#') or not line: - continue - line = line.split() - if len(line) not in (3,4): - raise Exception("Format should be: %s; line %s" % (format_string, line)) - options = {} - if len(line) == 4: - option_string = line.pop().rstrip(',') - try: - options = dict([[j.strip() for j in i.split('=', 1)] - for i in option_string.split(',')]) - except: - raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) + for manifest in manifests: + for line in file(i).readlines(): + line = line.strip() + if line.startswith('#') or not line: + continue + line = line.split() + if len(line) not in (3,4): + raise Exception("Format should be: %s; line %s" % (format_string, line)) + options = {} + if len(line) == 4: + option_string = line.pop().rstrip(',') + try: + options = dict([[j.strip() for j in i.split('=', 1)] + for i in option_string.split(',')]) + except: + raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) - url, dest, _type = line - retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) - return retval + url, dest, _type = line + retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) + return retval def main(args=sys.argv[1:]): - # parse command line options - usage = '%prog [options] manifest [manifest] [...]' + # parse command line options + usage = '%prog [options] manifest [manifest] [...]' - # description formatter - class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): - def format_description(self, description): - if description: - return description + '\n' - else: - return '' + class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): + def format_description(self, description): + if description: + return description + '\n' + else: + return '' - parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) - parser.add_option('-o', '--output', - help="output relative to this location vs. the manifest location") - parser.add_option('-d', '--dest', - action='append', - help="output only these destinations") - parser.add_option('-s', '--strict', - action='store_true', default=False, - help="fail on error") - parser.add_option('--list-fetchers', dest='list_fetchers', - action='store_true', default=False, - help='list available fetchers and exit') - options, args = parser.parse_args(args) + parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) + parser.add_option('-o', '--output', + help="output relative to this location vs. the manifest location") + parser.add_option('-d', '--dest', + action='append', + help="output only these destinations") + parser.add_option('-s', '--strict', + action='store_true', default=False, + help="fail on error") + parser.add_option('--list-fetchers', dest='list_fetchers', + action='store_true', default=False, + help='list available fetchers and exit') + options, args = parser.parse_args(args) - if options.list_fetchers: - for name in sorted(fetchers.keys()): - print name - parser.exit() + if options.list_fetchers: + for name in sorted(fetchers.keys()): + print name + parser.exit() - if not args: - parser.print_help() - parser.exit() + if not args: + parser.print_help() + parser.exit() - items = read_manifests(*args) - fetch = Fetch(fetchers.values(), strict=options.strict) + items = read_manifests(*args) + fetch = Fetch(fetchers.values(), strict=options.strict) - # download the files - fetch.fetch(*items) + # download the files + fetch.fetch(*items) if __name__ == '__main__': - main() + main()