Mercurial > hg > fetch
annotate fetch.py @ 22:63ff1b00ec05
first passing test
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Wed, 09 Nov 2011 20:11:05 -0800 |
| parents | 0706968f01bb |
| children | 12ad9ab11860 |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 """ | |
| 4 fetch stuff from the interwebs | |
| 5 """ | |
| 6 | |
| 7 import os | |
| 8 import sys | |
| 9 import optparse | |
| 10 | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
11 __all__ = ['Fetcher', 'Fetch', 'main'] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
12 |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
13 def which(executable, path=os.environ['PATH']): |
| 15 | 14 """python equivalent of which; should really be in the stdlib""" |
| 15 dirs = path.split(os.pathsep) | |
| 16 for dir in dirs: | |
| 17 if os.path.isfile(os.path.join(dir, executable)): | |
| 18 return os.path.join(dir, executable) | |
| 7 | 19 |
| 0 | 20 class Fetcher(object): |
| 15 | 21 """abstract base class for resource fetchers""" |
| 0 | 22 |
| 15 | 23 @classmethod |
| 24 def match(cls, _type): | |
| 25 return _type == cls.type | |
| 0 | 26 |
| 17 | 27 def __init__(self, url, clobber=False): |
| 28 self.subpath = None | |
| 29 if '#' in url: | |
| 30 url, self.subpath = url.rsplit('#') | |
| 15 | 31 self.url = url |
| 18 | 32 # self.clobber = clobber # unused |
| 0 | 33 |
| 15 | 34 def __call__(self, dest): |
| 17 | 35 raise NotImplementedError("Should be called by implementing class") |
| 36 | |
| 37 @classmethod | |
| 38 def doc(cls): | |
| 39 """return docstring for the instance""" | |
| 40 retval = getattr(cls, '__doc__', '').strip() | |
| 41 return ' '.join(retval.split()) | |
| 0 | 42 |
| 7 | 43 ### standard dispatchers - always available |
| 0 | 44 |
| 7 | 45 import tarfile |
| 0 | 46 import urllib2 |
| 7 | 47 from StringIO import StringIO |
| 0 | 48 |
| 5 | 49 class FileFetcher(Fetcher): |
| 15 | 50 """fetch a single file""" |
| 0 | 51 |
| 15 | 52 type = 'file' |
| 0 | 53 |
| 15 | 54 @classmethod |
| 55 def download(cls, url): | |
| 56 return urllib2.urlopen(url).read() | |
| 0 | 57 |
| 15 | 58 def __call__(self, dest): |
| 59 if os.path.isdir(dest): | |
| 60 filename = self.url.rsplit('/', 1)[-1] | |
| 61 dest = os.path.join(dest, filename) | |
| 62 f = file(dest, 'w') | |
| 63 f.write(self.download(self.url)) | |
| 64 f.close() | |
| 0 | 65 |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
66 |
| 5 | 67 class TarballFetcher(FileFetcher): |
| 15 | 68 """fetch and extract a tarball""" |
| 0 | 69 |
| 15 | 70 type = 'tar' |
| 0 | 71 |
| 15 | 72 def __call__(self, dest): |
| 73 assert os.path.isdir(dest) | |
| 17 | 74 if self.subpath: |
| 75 raise NotImplementedError("should extract only a subpath of a tarball but I haven't finished it yet") | |
| 15 | 76 buffer = StringIO() |
| 77 buffer.write(self.download(self.url)) | |
| 78 buffer.seek(0) | |
| 79 tf = tarfile.open(mode='r', fileobj=buffer) | |
| 80 tf.extract(dest) | |
| 7 | 81 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
82 fetchers = [FileFetcher, TarballFetcher] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
83 |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
84 ### VCS fetchers using executable |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
85 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
86 import subprocess |
| 19 | 87 try: |
| 88 from subprocess import check_call as call | |
| 89 except ImportErorr: | |
| 90 raise # we need check_call, kinda | |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
91 |
| 17 | 92 class VCSFetcher(Fetcher): |
| 93 def __init__(self, url, export=True): | |
| 94 """ | |
| 95 - export : whether to strip the versioning information | |
| 96 """ | |
| 97 Fetcher.__init__(self, url) | |
| 98 self.export = export | |
| 99 | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
100 if which('hg'): |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
101 |
| 17 | 102 class HgFetcher(VCSFetcher): |
| 15 | 103 """checkout a mercurial repository""" |
| 104 type = 'hg' | |
| 0 | 105 |
| 19 | 106 def __init__(self, url, export=True): |
| 107 VCSFetcher.__init__(self, url, export=True) | |
| 108 self.hg = which('hg') | |
| 109 | |
| 15 | 110 def __call__(self, dest): |
| 111 if os.path.exits(dest): | |
| 112 assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg')) | |
| 19 | 113 call([self.hg, 'pull', self.url], cwd=dest) |
| 114 call([self.hg, 'update'], cwd=dest) | |
| 115 else: | |
| 116 pass | |
| 17 | 117 raise NotImplementedError("TODO! Sorry!") |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
118 |
| 15 | 119 fetchers.append(HgFetcher) |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
120 |
| 15 | 121 if which('git'): |
| 17 | 122 |
| 15 | 123 class GitFetcher(Fetcher): |
| 124 """checkout a git repository""" | |
| 125 type = 'git' | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
126 |
| 20 | 127 def __init__(self, url, export=True): |
| 128 VCSFetcher.__init__(self, url, export=True) | |
| 129 self.hg = which('git') | |
| 130 | |
| 131 fetchers.append(GitFetcher) | |
| 17 | 132 |
| 16 | 133 __all__ += [i.__name__ for i in fetchers] |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
134 |
| 0 | 135 class Fetch(object): |
| 136 | |
| 21 | 137 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True): |
| 15 | 138 self.fetchers = fetchers |
| 139 self.relative_to = relative_to | |
| 140 self.strict = strict | |
| 0 | 141 |
| 15 | 142 def fetcher(self, _type): |
| 143 """find the fetcher for the appropriate type""" | |
| 144 for fetcher in fetchers: | |
| 145 if fetcher.match(_type): | |
| 146 return fetcher | |
| 0 | 147 |
| 15 | 148 def __call__(self, url, destination, type, **options): |
| 149 fetcher = self.fetcher(type) | |
| 150 assert fetcher is not None, "No fetcher found for type '%s'" % type | |
| 151 fetcher = fetcher(url, **options) | |
| 152 fetcher(destination) | |
| 2 | 153 |
| 15 | 154 def fetch(self, *items): |
| 2 | 155 |
| 15 | 156 if self.strict: |
| 157 # ensure all the required fetchers are available | |
| 158 types = set([i['type'] for i in items]) | |
| 159 assert not [i for i in types | |
| 160 if [True for fetcher in fetchers if fetcher.match(i)]] | |
| 4 | 161 |
| 15 | 162 for item in items: |
| 4 | 163 |
| 15 | 164 # fix up relative paths |
| 165 dest = item['dest'] | |
| 166 if not os.path.isabs(dest): | |
| 167 relative_to = self.relative_to or os.path.dirname(os.path.abspath(item['manifest'])) | |
| 168 dest = os.path.join(relative_to, dest) | |
| 4 | 169 |
| 15 | 170 # fetch the items |
| 171 self(item['url'], destination=dest, type=item['type'], **item['options']) | |
| 0 | 172 |
| 21 | 173 |
| 0 | 174 format_string = "[URL] [destination] [type] <options>" |
| 175 def read_manifests(*manifests): | |
| 15 | 176 """ |
| 177 read some manifests and return the items | |
| 178 | |
| 179 Format: | |
| 180 %s | |
| 181 """ % format_string | |
| 0 | 182 |
| 15 | 183 # sanity check |
| 184 assert not [i for i in manifests if not os.path.exists(i)] | |
| 0 | 185 |
| 15 | 186 retval = [] |
| 0 | 187 |
| 15 | 188 for manifest in manifests: |
| 189 for line in file(i).readlines(): | |
| 190 line = line.strip() | |
| 191 if line.startswith('#') or not line: | |
| 192 continue | |
| 193 line = line.split() | |
| 194 if len(line) not in (3,4): | |
| 195 raise Exception("Format should be: %s; line %s" % (format_string, line)) | |
| 196 options = {} | |
| 197 if len(line) == 4: | |
| 198 option_string = line.pop().rstrip(',') | |
| 199 try: | |
| 200 options = dict([[j.strip() for j in i.split('=', 1)] | |
| 201 for i in option_string.split(',')]) | |
| 202 except: | |
| 203 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) | |
| 0 | 204 |
| 15 | 205 url, dest, _type = line |
| 206 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) | |
| 207 return retval | |
| 0 | 208 |
| 2 | 209 def main(args=sys.argv[1:]): |
| 0 | 210 |
| 15 | 211 # parse command line options |
| 212 usage = '%prog [options] manifest [manifest] [...]' | |
| 0 | 213 |
| 15 | 214 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): |
| 215 def format_description(self, description): | |
| 216 if description: | |
| 217 return description + '\n' | |
| 218 else: | |
| 219 return '' | |
| 0 | 220 |
| 15 | 221 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) |
| 222 parser.add_option('-o', '--output', | |
| 223 help="output relative to this location vs. the manifest location") | |
| 17 | 224 parser.add_option('-d', '--dest', # XXX unused |
| 15 | 225 action='append', |
| 226 help="output only these destinations") | |
| 227 parser.add_option('-s', '--strict', | |
| 228 action='store_true', default=False, | |
| 229 help="fail on error") | |
| 230 parser.add_option('--list-fetchers', dest='list_fetchers', | |
| 231 action='store_true', default=False, | |
| 232 help='list available fetchers and exit') | |
| 233 options, args = parser.parse_args(args) | |
| 0 | 234 |
| 15 | 235 if options.list_fetchers: |
| 17 | 236 types = set() |
| 237 for fetcher in fetchers: | |
| 238 if fetcher.type in types: | |
| 239 continue # occluded, should probably display separately | |
| 240 print '%s : %s' % (fetcher.type, fetcher.doc()) | |
| 241 types.add(fetcher.type) | |
| 15 | 242 parser.exit() |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
243 |
| 15 | 244 if not args: |
| 17 | 245 # TODO: could read from stdin |
| 15 | 246 parser.print_help() |
| 247 parser.exit() | |
| 0 | 248 |
| 15 | 249 items = read_manifests(*args) |
| 16 | 250 fetch = Fetch(fetchers, strict=options.strict) |
| 0 | 251 |
| 15 | 252 # download the files |
| 253 fetch.fetch(*items) | |
| 0 | 254 |
| 255 if __name__ == '__main__': | |
| 15 | 256 main() |
| 0 | 257 |