Mercurial > hg > fetch
annotate fetch.py @ 25:e628ce3ae49f
more stubbing
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Mon, 14 Nov 2011 20:19:57 -0800 |
| parents | b1f65f3bd1bc |
| children | d495b610046a |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 """ | |
| 4 fetch stuff from the interwebs | |
| 5 """ | |
| 6 | |
| 7 import os | |
| 23 | 8 import shutil |
| 0 | 9 import sys |
| 10 import optparse | |
| 11 | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
12 __all__ = ['Fetcher', 'Fetch', 'main'] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
13 |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
14 def which(executable, path=os.environ['PATH']): |
| 15 | 15 """python equivalent of which; should really be in the stdlib""" |
| 16 dirs = path.split(os.pathsep) | |
| 17 for dir in dirs: | |
| 18 if os.path.isfile(os.path.join(dir, executable)): | |
| 19 return os.path.join(dir, executable) | |
| 7 | 20 |
| 0 | 21 class Fetcher(object): |
| 15 | 22 """abstract base class for resource fetchers""" |
| 0 | 23 |
| 15 | 24 @classmethod |
| 25 def match(cls, _type): | |
| 26 return _type == cls.type | |
| 0 | 27 |
| 23 | 28 def __init__(self, url, clobber=True): |
| 17 | 29 self.subpath = None |
| 30 if '#' in url: | |
| 31 url, self.subpath = url.rsplit('#') | |
| 25 | 32 if self.subpath: |
| 33 self.subpath = self.subpath.split('/') | |
| 15 | 34 self.url = url |
| 23 | 35 self.clobber = clobber |
| 0 | 36 |
| 15 | 37 def __call__(self, dest): |
| 17 | 38 raise NotImplementedError("Should be called by implementing class") |
| 39 | |
| 40 @classmethod | |
| 41 def doc(cls): | |
| 42 """return docstring for the instance""" | |
| 43 retval = getattr(cls, '__doc__', '').strip() | |
| 44 return ' '.join(retval.split()) | |
| 0 | 45 |
| 7 | 46 ### standard dispatchers - always available |
| 0 | 47 |
| 7 | 48 import tarfile |
| 0 | 49 import urllib2 |
| 7 | 50 from StringIO import StringIO |
| 0 | 51 |
| 5 | 52 class FileFetcher(Fetcher): |
| 15 | 53 """fetch a single file""" |
| 25 | 54 # Note: subpath for files is ignored |
| 0 | 55 |
| 15 | 56 type = 'file' |
| 0 | 57 |
| 15 | 58 @classmethod |
| 59 def download(cls, url): | |
| 60 return urllib2.urlopen(url).read() | |
| 0 | 61 |
| 15 | 62 def __call__(self, dest): |
| 25 | 63 |
| 15 | 64 if os.path.isdir(dest): |
| 65 filename = self.url.rsplit('/', 1)[-1] | |
| 66 dest = os.path.join(dest, filename) | |
| 67 f = file(dest, 'w') | |
| 68 f.write(self.download(self.url)) | |
| 69 f.close() | |
| 0 | 70 |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
71 |
| 5 | 72 class TarballFetcher(FileFetcher): |
| 15 | 73 """fetch and extract a tarball""" |
| 0 | 74 |
| 15 | 75 type = 'tar' |
| 0 | 76 |
| 15 | 77 def __call__(self, dest): |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
78 if os.path.exists(dest): |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
79 assert os.path.isdir(dest) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
80 else: |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
81 os.mkdirs(dest) |
| 17 | 82 if self.subpath: |
| 83 raise NotImplementedError("should extract only a subpath of a tarball but I haven't finished it yet") | |
| 15 | 84 buffer = StringIO() |
| 85 buffer.write(self.download(self.url)) | |
| 86 buffer.seek(0) | |
| 87 tf = tarfile.open(mode='r', fileobj=buffer) | |
| 88 tf.extract(dest) | |
| 7 | 89 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
90 fetchers = [FileFetcher, TarballFetcher] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
91 |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
92 ### VCS fetchers |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
93 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
94 import subprocess |
| 19 | 95 try: |
| 96 from subprocess import check_call as call | |
| 97 except ImportErorr: | |
| 98 raise # we need check_call, kinda | |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
99 |
| 17 | 100 class VCSFetcher(Fetcher): |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
101 |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
102 command = None |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
103 |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
104 def call(*args, **kwargs): |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
105 assert command is not None, "Abstract base class" |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
106 call([self.command] + list(args), **kwargs) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
107 |
| 17 | 108 def __init__(self, url, export=True): |
| 109 """ | |
| 110 - export : whether to strip the versioning information | |
| 111 """ | |
| 112 Fetcher.__init__(self, url) | |
| 113 self.export = export | |
| 114 | |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
115 def __call__(self, dest): |
| 25 | 116 |
| 117 if self.subpath or self.export: | |
| 118 # can only export with a subpath | |
| 119 | |
| 120 | |
| 121 if os.path.exists(dest): | |
| 122 assert os.path.isdir(dest) | |
| 123 | |
| 124 def export(self, dest): | |
| 125 """ | |
| 126 export a clone of the directory | |
| 127 """ | |
| 128 tmpdir = tempfile.mkdtmp() | |
| 129 self.clone(tmpdir) | |
| 130 | |
| 131 shutil.rmtree(tmpdir) | |
| 132 | |
| 133 def clone(self, dest): | |
| 134 """ | |
| 135 clones into a directory | |
| 136 """ | |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
137 raise NotImplementedError("Abstract base class") |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
138 |
| 25 | 139 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
140 if which('hg'): |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
141 |
| 17 | 142 class HgFetcher(VCSFetcher): |
| 15 | 143 """checkout a mercurial repository""" |
| 144 type = 'hg' | |
| 0 | 145 |
| 19 | 146 def __init__(self, url, export=True): |
| 147 VCSFetcher.__init__(self, url, export=True) | |
| 148 self.hg = which('hg') | |
| 25 | 149 assert self.hg, "'hg' command not found" |
| 19 | 150 |
| 15 | 151 def __call__(self, dest): |
| 23 | 152 if os.path.exists(dest): |
| 15 | 153 assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg')) |
| 19 | 154 call([self.hg, 'pull', self.url], cwd=dest) |
| 155 call([self.hg, 'update'], cwd=dest) | |
| 156 else: | |
| 23 | 157 if not os.path.exists(dest): |
| 158 os.mkdirs(dest) | |
| 159 call([self.hg, 'clone', self.url, dest]) | |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
160 |
| 15 | 161 fetchers.append(HgFetcher) |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
162 |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
163 |
| 15 | 164 if which('git'): |
| 17 | 165 |
| 15 | 166 class GitFetcher(Fetcher): |
| 167 """checkout a git repository""" | |
| 168 type = 'git' | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
169 |
| 20 | 170 def __init__(self, url, export=True): |
| 171 VCSFetcher.__init__(self, url, export=True) | |
| 23 | 172 self.git = which('git') |
| 173 | |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
174 def __call__(self, dest): |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
175 if os.path.exists(dest): |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
176 assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.git')) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
177 call([self.git, 'pull', self.url], cwd=dest) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
178 call([self.hg, 'update'], cwd=dest) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
179 else: |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
180 if not os.path.exists(dest): |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
181 os.mkdirs(dest) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
182 call([self.hg, 'clone', self.url, dest]) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
183 |
| 20 | 184 |
| 185 fetchers.append(GitFetcher) | |
| 17 | 186 |
| 16 | 187 __all__ += [i.__name__ for i in fetchers] |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
188 |
| 0 | 189 class Fetch(object): |
| 190 | |
| 21 | 191 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True): |
| 15 | 192 self.fetchers = fetchers |
| 193 self.relative_to = relative_to | |
| 194 self.strict = strict | |
| 0 | 195 |
| 15 | 196 def fetcher(self, _type): |
| 197 """find the fetcher for the appropriate type""" | |
| 198 for fetcher in fetchers: | |
| 199 if fetcher.match(_type): | |
| 200 return fetcher | |
| 0 | 201 |
| 15 | 202 def __call__(self, url, destination, type, **options): |
| 203 fetcher = self.fetcher(type) | |
| 204 assert fetcher is not None, "No fetcher found for type '%s'" % type | |
| 205 fetcher = fetcher(url, **options) | |
| 206 fetcher(destination) | |
| 2 | 207 |
| 15 | 208 def fetch(self, *items): |
| 2 | 209 |
| 15 | 210 if self.strict: |
| 211 # ensure all the required fetchers are available | |
| 212 types = set([i['type'] for i in items]) | |
| 213 assert not [i for i in types | |
| 214 if [True for fetcher in fetchers if fetcher.match(i)]] | |
| 4 | 215 |
| 15 | 216 for item in items: |
| 4 | 217 |
| 15 | 218 # fix up relative paths |
| 219 dest = item['dest'] | |
| 220 if not os.path.isabs(dest): | |
| 221 relative_to = self.relative_to or os.path.dirname(os.path.abspath(item['manifest'])) | |
| 222 dest = os.path.join(relative_to, dest) | |
| 4 | 223 |
| 15 | 224 # fetch the items |
| 225 self(item['url'], destination=dest, type=item['type'], **item['options']) | |
| 0 | 226 |
| 21 | 227 |
| 0 | 228 format_string = "[URL] [destination] [type] <options>" |
| 229 def read_manifests(*manifests): | |
| 15 | 230 """ |
| 231 read some manifests and return the items | |
| 232 | |
| 233 Format: | |
| 234 %s | |
| 235 """ % format_string | |
| 0 | 236 |
| 15 | 237 # sanity check |
| 238 assert not [i for i in manifests if not os.path.exists(i)] | |
| 0 | 239 |
| 15 | 240 retval = [] |
| 0 | 241 |
| 15 | 242 for manifest in manifests: |
| 243 for line in file(i).readlines(): | |
| 244 line = line.strip() | |
| 245 if line.startswith('#') or not line: | |
| 246 continue | |
| 247 line = line.split() | |
| 248 if len(line) not in (3,4): | |
| 249 raise Exception("Format should be: %s; line %s" % (format_string, line)) | |
| 250 options = {} | |
| 251 if len(line) == 4: | |
| 252 option_string = line.pop().rstrip(',') | |
| 253 try: | |
| 254 options = dict([[j.strip() for j in i.split('=', 1)] | |
| 255 for i in option_string.split(',')]) | |
| 256 except: | |
| 257 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) | |
| 0 | 258 |
| 15 | 259 url, dest, _type = line |
| 260 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) | |
| 261 return retval | |
| 0 | 262 |
| 2 | 263 def main(args=sys.argv[1:]): |
| 0 | 264 |
| 15 | 265 # parse command line options |
| 266 usage = '%prog [options] manifest [manifest] [...]' | |
| 0 | 267 |
| 15 | 268 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): |
| 269 def format_description(self, description): | |
| 270 if description: | |
| 271 return description + '\n' | |
| 272 else: | |
| 273 return '' | |
| 0 | 274 |
| 15 | 275 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) |
| 276 parser.add_option('-o', '--output', | |
| 277 help="output relative to this location vs. the manifest location") | |
| 17 | 278 parser.add_option('-d', '--dest', # XXX unused |
| 15 | 279 action='append', |
| 280 help="output only these destinations") | |
| 281 parser.add_option('-s', '--strict', | |
| 282 action='store_true', default=False, | |
| 283 help="fail on error") | |
| 284 parser.add_option('--list-fetchers', dest='list_fetchers', | |
| 285 action='store_true', default=False, | |
| 286 help='list available fetchers and exit') | |
| 287 options, args = parser.parse_args(args) | |
| 0 | 288 |
| 15 | 289 if options.list_fetchers: |
| 17 | 290 types = set() |
| 291 for fetcher in fetchers: | |
| 292 if fetcher.type in types: | |
| 293 continue # occluded, should probably display separately | |
| 294 print '%s : %s' % (fetcher.type, fetcher.doc()) | |
| 295 types.add(fetcher.type) | |
| 15 | 296 parser.exit() |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
297 |
| 15 | 298 if not args: |
| 17 | 299 # TODO: could read from stdin |
| 15 | 300 parser.print_help() |
| 301 parser.exit() | |
| 0 | 302 |
| 15 | 303 items = read_manifests(*args) |
| 16 | 304 fetch = Fetch(fetchers, strict=options.strict) |
| 0 | 305 |
| 15 | 306 # download the files |
| 307 fetch.fetch(*items) | |
| 0 | 308 |
| 309 if __name__ == '__main__': | |
| 15 | 310 main() |
| 0 | 311 |