Mercurial > hg > fetch
annotate fetch.py @ 14:626b027134ea
flush out example file
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Wed, 09 Nov 2011 16:20:30 -0800 |
| parents | 3fee8ecd1af8 |
| children | bc7d6763357e |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 """ | |
| 4 fetch stuff from the interwebs | |
| 5 """ | |
| 6 | |
| 7 import os | |
| 8 import sys | |
| 9 import optparse | |
| 10 | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
11 __all__ = ['Fetcher', 'Fetch', 'main'] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
12 |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
13 def which(executable, path=os.environ['PATH']): |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
14 """python equivalent of which; should really be in the stdlib""" |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
15 # XXX from https://github.com/mozautomation/mozmill/blob/master/mozrunner/mozrunner/utils.py |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
16 dirs = path.split(os.pathsep) |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
17 for dir in dirs: |
| 9 | 18 if os.path.isfile(os.path.join(dir, executable)): |
| 19 return os.path.join(dir, executable) | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
20 |
| 7 | 21 |
| 0 | 22 class Fetcher(object): |
| 23 """abstract base class for resource fetchers""" | |
| 24 | |
| 25 @classmethod | |
| 26 def match(cls, _type): | |
| 27 return _type == cls.type | |
| 28 | |
| 29 def __init__(self, url): | |
| 30 self.url = url | |
| 31 | |
| 32 def __call__(self, dest): | |
| 33 raise NotImplementedError | |
| 34 | |
| 7 | 35 ### standard dispatchers - always available |
| 0 | 36 |
| 7 | 37 import tarfile |
| 0 | 38 import urllib2 |
| 7 | 39 from StringIO import StringIO |
| 0 | 40 |
| 5 | 41 class FileFetcher(Fetcher): |
| 0 | 42 """fetch a single file""" |
| 43 | |
| 44 type = 'file' | |
| 45 | |
| 46 @classmethod | |
| 47 def download(cls, url): | |
| 48 return urllib2.urlopen(url).read() | |
| 49 | |
| 50 def __call__(self, dest): | |
| 51 if os.path.isdir(dest): | |
| 5 | 52 filename = self.url.rsplit('/', 1)[-1] |
| 0 | 53 dest = os.path.join(dest, filename) |
| 54 f = file(dest, 'w') | |
| 55 f.write(self.download(self.url)) | |
| 56 f.close() | |
| 57 | |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
58 |
| 5 | 59 class TarballFetcher(FileFetcher): |
| 0 | 60 """fetch and extract a tarball""" |
| 61 | |
| 62 type = 'tar' | |
| 63 | |
| 7 | 64 def __call__(self, dest): |
| 65 assert os.path.isdir(dest) | |
| 66 buffer = StringIO() | |
| 67 buffer.write(self.download(self.url)) | |
| 68 buffer.seek(0) | |
| 69 tf = tarfile.open(mode='r', fileobj=buffer) | |
| 70 tf.extract(dest) | |
| 71 | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
72 fetchers = [FileFetcher, TarballFetcher] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
73 |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
74 ### VCS fetchers using executable |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
75 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
76 import subprocess |
|
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
77 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
78 if which('hg'): |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
79 |
|
10
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
80 class HgFetcher(Fetcher): |
|
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
81 """checkout a mercurial repository""" |
|
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
82 type = 'hg' |
| 0 | 83 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
84 def __call__(self, dest): |
|
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
85 if os.path.exits(dest): |
|
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
86 assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg')) |
|
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
87 pass # TODO |
|
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
88 |
|
10
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
89 fetchers.append(HgFetcher) |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
90 |
| 5 | 91 class GitFetcher(Fetcher): |
| 0 | 92 """checkout a git repository""" |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
93 type = 'git' |
|
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
94 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
95 |
|
10
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
96 fetchers = dict([(i.__name__, i) for i in fetchers]) |
|
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
97 __all__ += fetchers.keys() |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
98 |
| 0 | 99 |
| 100 class Fetch(object): | |
| 101 | |
| 102 def __init__(self, fetchers, relative_to=None, strict=True): | |
| 103 self.fetchers = fetchers | |
| 104 self.relative_to = relative_to | |
| 105 self.strict = strict | |
| 106 | |
| 107 def fetcher(self, _type): | |
| 108 """find the fetcher for the appropriate type""" | |
| 109 for fetcher in fetchers: | |
| 110 if fetcher.match(_type): | |
| 111 return fetcher | |
| 112 | |
| 113 def __call__(self, url, destination, type, **options): | |
| 114 fetcher = self.fetcher(type) | |
| 115 assert fetcher is not None | |
| 116 fetcher = fetcher(url, **options) | |
| 117 fetcher(destination) | |
| 118 | |
| 119 def fetch(self, *items): | |
| 2 | 120 |
| 121 if self.strict: | |
| 122 # ensure all the required fetchers are available | |
| 123 types = set([i['type'] for i in items]) | |
| 124 assert not [i for i in types | |
| 125 if [True for fetcher in fetchers if fetcher.match(i)]] | |
| 126 | |
| 0 | 127 for item in items: |
| 4 | 128 |
| 129 # fix up relative paths | |
| 5 | 130 dest = item['dest'] |
| 4 | 131 if not os.path.isabs(dest): |
| 132 if self.relative_to: | |
| 133 dest = os.path.join(self.relative_to, dest) | |
| 134 else: | |
| 135 dest = os.path.join(os.path.dirname(os.path.abspath(item['manifest'])), dest) | |
| 136 | |
| 137 # fetch the items | |
| 138 self(item['url'], destination=dest, type=item['type'], **item['options']) | |
| 139 | |
| 0 | 140 |
| 141 format_string = "[URL] [destination] [type] <options>" | |
| 142 def read_manifests(*manifests): | |
| 143 """ | |
| 144 read some manifests and return the items | |
| 145 | |
| 146 Format: | |
| 147 %s | |
| 148 """ % format_string | |
| 149 | |
| 150 # sanity check | |
| 2 | 151 assert not [i for i in manifests if not os.path.exists(i)] |
| 0 | 152 |
| 153 retval = [] | |
| 154 | |
| 155 for manifest in manifests: | |
| 156 for line in file(i).readlines(): | |
| 157 line = line.strip() | |
| 3 | 158 if line.startswith('#') or not line: |
| 159 continue | |
| 0 | 160 line = line.split() |
| 161 if len(line) not in (3,4): | |
| 162 raise Exception("Format should be: %s; line %s" % (format_string, line)) | |
| 163 options = {} | |
| 164 if len(line) == 4: | |
| 165 option_string = line.pop().rstrip(',') | |
| 166 try: | |
| 167 options = dict([[j.strip() for j in i.split('=', 1)] | |
| 168 for i in option_string.split(',')]) | |
| 169 except: | |
| 170 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) | |
| 171 | |
| 172 url, dest, _type = line | |
| 173 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) | |
| 174 return retval | |
| 175 | |
| 2 | 176 def main(args=sys.argv[1:]): |
| 0 | 177 |
| 178 # parse command line options | |
| 179 usage = '%prog [options] manifest [manifest] [...]' | |
| 180 | |
| 181 # description formatter | |
| 182 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): | |
| 183 def format_description(self, description): | |
| 184 if description: | |
| 185 return description + '\n' | |
| 186 else: | |
| 187 return '' | |
| 188 | |
| 189 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) | |
| 190 parser.add_option('-o', '--output', | |
| 191 help="output relative to this location vs. the manifest location") | |
| 192 parser.add_option('-d', '--dest', | |
| 193 action='append', | |
| 194 help="output only these destinations") | |
| 195 parser.add_option('-s', '--strict', | |
| 196 action='store_true', default=False, | |
| 197 help="fail on error") | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
198 parser.add_option('--list-fetchers', dest='list_fetchers', |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
199 action='store_true', default=False, |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
200 help='list available fetchers and exit') |
| 0 | 201 options, args = parser.parse_args(args) |
| 202 | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
203 if options.list_fetchers: |
|
10
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
204 for name in sorted(fetchers.keys()): |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
205 print name |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
206 parser.exit() |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
207 |
| 0 | 208 if not args: |
| 209 parser.print_help() | |
| 210 parser.exit() | |
| 211 | |
| 212 items = read_manifests(*args) | |
|
10
0b534c8881de
make fetchers a dict keyed on class name
Jeff Hammel <jhammel@mozilla.com>
parents:
9
diff
changeset
|
213 fetch = Fetch(fetchers.values(), strict=options.strict) |
| 0 | 214 |
| 215 # download the files | |
| 216 fetch.fetch(*items) | |
| 217 | |
| 218 if __name__ == '__main__': | |
| 219 main() | |
| 220 |