Mercurial > hg > fetch
annotate fetch/main.py @ 8:cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
author | Jeff Hammel <jhammel@mozilla.com> |
---|---|
date | Sun, 18 Sep 2011 18:46:31 -0700 |
parents | 4d3d0e1324f5 |
children | f8575a78ec06 |
rev | line source |
---|---|
0 | 1 #!/usr/bin/env python |
2 | |
3 """ | |
4 fetch stuff from the interwebs | |
5 """ | |
6 | |
7 import os | |
8 import sys | |
9 import optparse | |
10 | |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
11 __all__ = ['Fetcher', 'Fetch', 'main'] |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
12 |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
13 def which(executable, path=os.environ['PATH']): |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
14 """python equivalent of which; should really be in the stdlib""" |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
15 # XXX from https://github.com/mozautomation/mozmill/blob/master/mozrunner/mozrunner/utils.py |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
16 dirs = path.split(os.pathsep) |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
17 for dir in dirs: |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
18 if os.path.isfile(os.path.join(dir, fileName)): |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
19 return os.path.join(dir, fileName) |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
20 |
7 | 21 |
0 | 22 class Fetcher(object): |
23 """abstract base class for resource fetchers""" | |
24 | |
25 @classmethod | |
26 def match(cls, _type): | |
27 return _type == cls.type | |
28 | |
29 def __init__(self, url): | |
30 self.url = url | |
31 | |
32 def __call__(self, dest): | |
33 raise NotImplementedError | |
34 | |
7 | 35 ### standard dispatchers - always available |
0 | 36 |
7 | 37 import tarfile |
0 | 38 import urllib2 |
7 | 39 from StringIO import StringIO |
0 | 40 |
5 | 41 class FileFetcher(Fetcher): |
0 | 42 """fetch a single file""" |
43 | |
44 type = 'file' | |
45 | |
46 @classmethod | |
47 def download(cls, url): | |
48 return urllib2.urlopen(url).read() | |
49 | |
50 def __call__(self, dest): | |
51 if os.path.isdir(dest): | |
5 | 52 filename = self.url.rsplit('/', 1)[-1] |
0 | 53 dest = os.path.join(dest, filename) |
54 f = file(dest, 'w') | |
55 f.write(self.download(self.url)) | |
56 f.close() | |
57 | |
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
58 |
5 | 59 class TarballFetcher(FileFetcher): |
0 | 60 """fetch and extract a tarball""" |
61 | |
62 type = 'tar' | |
63 | |
7 | 64 def __call__(self, dest): |
65 assert os.path.isdir(dest) | |
66 buffer = StringIO() | |
67 buffer.write(self.download(self.url)) | |
68 buffer.seek(0) | |
69 tf = tarfile.open(mode='r', fileobj=buffer) | |
70 tf.extract(dest) | |
71 | |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
72 fetchers = [FileFetcher, TarballFetcher] |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
73 |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
74 ### VCS fetchers using executable |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
75 |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
76 if which('hg'): |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
77 pass # TODO: wrap HgFetcher |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
78 |
5 | 79 class HgFetcher(Fetcher): |
0 | 80 """checkout a mercurial repository""" |
81 | |
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
82 type = 'hg' |
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
83 |
5 | 84 class GitFetcher(Fetcher): |
0 | 85 """checkout a git repository""" |
86 | |
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
87 type = 'git' |
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
88 |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
89 |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
90 fetcher_names = [cls.__name__ for cls in fetchers] |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
91 __all__ += fetcher_names |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
92 |
0 | 93 |
94 class Fetch(object): | |
95 | |
96 def __init__(self, fetchers, relative_to=None, strict=True): | |
97 self.fetchers = fetchers | |
98 self.relative_to = relative_to | |
99 self.strict = strict | |
100 | |
101 def fetcher(self, _type): | |
102 """find the fetcher for the appropriate type""" | |
103 for fetcher in fetchers: | |
104 if fetcher.match(_type): | |
105 return fetcher | |
106 | |
107 def __call__(self, url, destination, type, **options): | |
108 fetcher = self.fetcher(type) | |
109 assert fetcher is not None | |
110 fetcher = fetcher(url, **options) | |
111 fetcher(destination) | |
112 | |
113 def fetch(self, *items): | |
2 | 114 |
115 if self.strict: | |
116 # ensure all the required fetchers are available | |
117 types = set([i['type'] for i in items]) | |
118 assert not [i for i in types | |
119 if [True for fetcher in fetchers if fetcher.match(i)]] | |
120 | |
0 | 121 for item in items: |
4 | 122 |
123 # fix up relative paths | |
5 | 124 dest = item['dest'] |
4 | 125 if not os.path.isabs(dest): |
126 if self.relative_to: | |
127 dest = os.path.join(self.relative_to, dest) | |
128 else: | |
129 dest = os.path.join(os.path.dirname(os.path.abspath(item['manifest'])), dest) | |
130 | |
131 # fetch the items | |
132 self(item['url'], destination=dest, type=item['type'], **item['options']) | |
133 | |
0 | 134 |
135 format_string = "[URL] [destination] [type] <options>" | |
136 def read_manifests(*manifests): | |
137 """ | |
138 read some manifests and return the items | |
139 | |
140 Format: | |
141 %s | |
142 """ % format_string | |
143 | |
144 # sanity check | |
2 | 145 assert not [i for i in manifests if not os.path.exists(i)] |
0 | 146 |
147 retval = [] | |
148 | |
149 for manifest in manifests: | |
150 for line in file(i).readlines(): | |
151 line = line.strip() | |
3 | 152 if line.startswith('#') or not line: |
153 continue | |
0 | 154 line = line.split() |
155 if len(line) not in (3,4): | |
156 raise Exception("Format should be: %s; line %s" % (format_string, line)) | |
157 options = {} | |
158 if len(line) == 4: | |
159 option_string = line.pop().rstrip(',') | |
160 try: | |
161 options = dict([[j.strip() for j in i.split('=', 1)] | |
162 for i in option_string.split(',')]) | |
163 except: | |
164 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) | |
165 | |
166 url, dest, _type = line | |
167 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) | |
168 return retval | |
169 | |
2 | 170 def main(args=sys.argv[1:]): |
0 | 171 |
172 # parse command line options | |
173 usage = '%prog [options] manifest [manifest] [...]' | |
174 | |
175 # description formatter | |
176 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): | |
177 def format_description(self, description): | |
178 if description: | |
179 return description + '\n' | |
180 else: | |
181 return '' | |
182 | |
183 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) | |
184 parser.add_option('-o', '--output', | |
185 help="output relative to this location vs. the manifest location") | |
186 parser.add_option('-d', '--dest', | |
187 action='append', | |
188 help="output only these destinations") | |
189 parser.add_option('-s', '--strict', | |
190 action='store_true', default=False, | |
191 help="fail on error") | |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
192 parser.add_option('--list-fetchers', dest='list_fetchers', |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
193 action='store_true', default=False, |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
194 help='list available fetchers and exit') |
0 | 195 options, args = parser.parse_args(args) |
196 | |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
197 if options.list_fetchers: |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
198 for name in fetcher_names: |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
199 print name |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
200 parser.exit() |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
201 |
0 | 202 if not args: |
203 parser.print_help() | |
204 parser.exit() | |
205 | |
206 items = read_manifests(*args) | |
207 fetch = Fetch(fetchers, strict=options.strict) | |
208 | |
209 # download the files | |
210 fetch.fetch(*items) | |
211 | |
212 if __name__ == '__main__': | |
213 main() | |
214 |