Mercurial > hg > fetch
annotate fetch.py @ 38:645d02834042
extract....correctly?
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Tue, 15 Nov 2011 12:56:35 -0800 |
| parents | f30fe9183e64 |
| children | 8addc3712e75 |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 """ | |
| 4 fetch stuff from the interwebs | |
| 5 """ | |
| 6 | |
| 7 import os | |
| 23 | 8 import shutil |
| 0 | 9 import sys |
| 10 import optparse | |
| 11 | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
12 __all__ = ['Fetcher', 'Fetch', 'main'] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
13 |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
14 def which(executable, path=os.environ['PATH']): |
| 15 | 15 """python equivalent of which; should really be in the stdlib""" |
| 16 dirs = path.split(os.pathsep) | |
| 17 for dir in dirs: | |
| 18 if os.path.isfile(os.path.join(dir, executable)): | |
| 19 return os.path.join(dir, executable) | |
| 7 | 20 |
| 0 | 21 class Fetcher(object): |
| 15 | 22 """abstract base class for resource fetchers""" |
| 0 | 23 |
| 15 | 24 @classmethod |
| 25 def match(cls, _type): | |
| 26 return _type == cls.type | |
| 0 | 27 |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
28 def __init__(self, url): |
| 17 | 29 self.subpath = None |
| 30 if '#' in url: | |
| 31 url, self.subpath = url.rsplit('#') | |
| 25 | 32 if self.subpath: |
| 33 self.subpath = self.subpath.split('/') | |
| 15 | 34 self.url = url |
| 0 | 35 |
| 15 | 36 def __call__(self, dest): |
| 17 | 37 raise NotImplementedError("Should be called by implementing class") |
| 38 | |
| 39 @classmethod | |
| 40 def doc(cls): | |
| 41 """return docstring for the instance""" | |
| 42 retval = getattr(cls, '__doc__', '').strip() | |
| 43 return ' '.join(retval.split()) | |
| 0 | 44 |
| 7 | 45 ### standard dispatchers - always available |
| 0 | 46 |
| 7 | 47 import tarfile |
| 0 | 48 import urllib2 |
| 7 | 49 from StringIO import StringIO |
| 0 | 50 |
| 5 | 51 class FileFetcher(Fetcher): |
| 15 | 52 """fetch a single file""" |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
53 # Note: subpath for single files is ignored |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
54 |
| 15 | 55 type = 'file' |
| 0 | 56 |
| 15 | 57 @classmethod |
| 58 def download(cls, url): | |
| 59 return urllib2.urlopen(url).read() | |
| 0 | 60 |
| 15 | 61 def __call__(self, dest): |
| 25 | 62 |
| 15 | 63 if os.path.isdir(dest): |
| 64 filename = self.url.rsplit('/', 1)[-1] | |
| 65 dest = os.path.join(dest, filename) | |
| 66 f = file(dest, 'w') | |
| 67 f.write(self.download(self.url)) | |
| 68 f.close() | |
| 0 | 69 |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
70 |
| 5 | 71 class TarballFetcher(FileFetcher): |
| 15 | 72 """fetch and extract a tarball""" |
| 0 | 73 |
| 15 | 74 type = 'tar' |
| 0 | 75 |
| 15 | 76 def __call__(self, dest): |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
77 if os.path.exists(dest): |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
78 assert os.path.isdir(dest), "Destination must be a directory" |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
79 else: |
| 35 | 80 os.makedirs(dest) |
| 17 | 81 if self.subpath: |
| 82 raise NotImplementedError("should extract only a subpath of a tarball but I haven't finished it yet") | |
| 15 | 83 buffer = StringIO() |
| 84 buffer.write(self.download(self.url)) | |
| 85 buffer.seek(0) | |
| 86 tf = tarfile.open(mode='r', fileobj=buffer) | |
| 38 | 87 members = tf.getmembers() |
| 88 for member in members: | |
| 89 tf.extract(member, dest) | |
| 7 | 90 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
91 fetchers = [FileFetcher, TarballFetcher] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
92 |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
93 ### VCS fetchers |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
94 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
95 import subprocess |
| 19 | 96 try: |
| 97 from subprocess import check_call as call | |
| 98 except ImportErorr: | |
| 99 raise # we need check_call, kinda | |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
100 |
| 17 | 101 class VCSFetcher(Fetcher): |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
102 |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
103 command = None |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
104 |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
105 def call(*args, **kwargs): |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
106 assert command is not None, "Abstract base class" |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
107 call([self.command] + list(args), **kwargs) |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
108 |
| 17 | 109 def __init__(self, url, export=True): |
| 110 """ | |
| 111 - export : whether to strip the versioning information | |
| 112 """ | |
| 113 Fetcher.__init__(self, url) | |
| 114 self.export = export | |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
115 self.prog = self.type # name of app program |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
116 self.vcs_dir = '.' + self.type # subdirectory for version control |
| 17 | 117 |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
118 def __call__(self, dest): |
| 25 | 119 |
| 120 if self.subpath or self.export: | |
| 121 # can only export with a subpath | |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
122 self.export(dest, subpath=self.subpath) |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
123 return |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
124 |
| 25 | 125 if os.path.exists(dest): |
| 126 assert os.path.isdir(dest) | |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
127 else: |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
128 self.clone(dest) |
| 25 | 129 |
| 130 def export(self, dest): | |
| 131 """ | |
| 132 export a clone of the directory | |
| 133 """ | |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
134 dest = os.path.abspath(dest) |
| 25 | 135 tmpdir = tempfile.mkdtmp() |
| 136 self.clone(tmpdir) | |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
137 path = tmpdir |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
138 if self.subpath: |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
139 path = os.path.join([tmpdir] + self.subpath) |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
140 assert os.path.exists(path), "subpath %s of %s not found" % (os.path.sep.join(self.subpath), self.url) |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
141 if os.path.isdir(path): |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
142 if os.path.exists(dest): |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
143 assert os.path.isdir(dest), "source is a directory; destination is a file" |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
144 else: |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
145 os.makedirs(dest) |
| 27 | 146 shutil.copytree(path, dest) |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
147 else: |
| 27 | 148 if not os.path.exists(dest): |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
149 directory, filename = os.path.split(dest) |
| 27 | 150 if os.path.exists(directory): |
| 151 assert os.path.isdir(directory), "%s is not a directory" % directory | |
| 152 else: | |
| 153 os.makedirs(directory) | |
| 154 shutil.copy(path, dest) | |
| 25 | 155 shutil.rmtree(tmpdir) |
| 156 | |
| 157 def clone(self, dest): | |
| 158 """ | |
| 159 clones into a directory | |
| 160 """ | |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
161 raise NotImplementedError("Abstract base class") |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
162 |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
163 def update(self, dest): |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
164 """ |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
165 updates a checkout |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
166 """ |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
167 raise NotImplementedError("Abstract base class") |
| 25 | 168 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
169 def versioned(self, directory): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
170 return os.path.exists(os.path.join(directory, self.vcs_dir)) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
171 |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
172 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
173 if which('hg'): |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
174 |
| 17 | 175 class HgFetcher(VCSFetcher): |
| 15 | 176 """checkout a mercurial repository""" |
| 177 type = 'hg' | |
| 0 | 178 |
| 19 | 179 def __init__(self, url, export=True): |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
180 VCSFetcher.__init__(self, url, export=export) |
| 19 | 181 self.hg = which('hg') |
| 25 | 182 assert self.hg, "'hg' command not found" |
| 19 | 183 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
184 def clone(self, dest): |
| 23 | 185 if os.path.exists(dest): |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
186 assert os.path.isdir(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
187 call([self.hg, 'clone', self.url, dest]) |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
188 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
189 def update(self, dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
190 assert os.path.versioned(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
191 assert os.path.exists(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
192 call([self.hg, 'pull', self.url], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
193 call([self.hg, 'update', '-C'], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
194 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
195 |
| 15 | 196 fetchers.append(HgFetcher) |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
197 |
| 15 | 198 if which('git'): |
| 17 | 199 |
| 15 | 200 class GitFetcher(Fetcher): |
| 201 """checkout a git repository""" | |
| 202 type = 'git' | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
203 |
| 20 | 204 def __init__(self, url, export=True): |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
205 VCSFetcher.__init__(self, url, export=export) |
| 23 | 206 self.git = which('git') |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
207 assert self.git, "'git' command not found" |
| 23 | 208 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
209 def update(self, dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
210 assert os.path.isdir(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
211 assert os.path.versioned(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
212 call([self.git, 'pull', self.url], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
213 call([self.git, 'update'], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
214 |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
215 def clone(self, dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
216 if not os.path.exists(dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
217 os.makedirs(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
218 call([self.git, 'clone', self.url, dest]) |
| 20 | 219 |
| 220 fetchers.append(GitFetcher) | |
| 17 | 221 |
| 16 | 222 __all__ += [i.__name__ for i in fetchers] |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
223 |
| 0 | 224 class Fetch(object): |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
225 |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
226 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True, clobber=True): |
| 15 | 227 self.fetchers = fetchers |
| 228 self.relative_to = relative_to | |
| 229 self.strict = strict | |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
230 self._clobber = clobber |
| 0 | 231 |
| 15 | 232 def fetcher(self, _type): |
| 233 """find the fetcher for the appropriate type""" | |
| 234 for fetcher in fetchers: | |
| 235 if fetcher.match(_type): | |
| 236 return fetcher | |
| 0 | 237 |
| 15 | 238 def __call__(self, url, destination, type, **options): |
| 239 fetcher = self.fetcher(type) | |
| 240 assert fetcher is not None, "No fetcher found for type '%s'" % type | |
| 241 fetcher = fetcher(url, **options) | |
| 242 fetcher(destination) | |
| 2 | 243 |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
244 def clobber(self, dest): |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
245 """clobbers if self._clobber is set""" |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
246 if self._clobber and os.path.exists(dest): |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
247 if os.path.isfile(dest): |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
248 os.remove(dest) |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
249 else: |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
250 shutil.rmtree(dest) |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
251 |
| 15 | 252 def fetch(self, *items): |
| 2 | 253 |
| 15 | 254 if self.strict: |
| 255 # ensure all the required fetchers are available | |
| 256 types = set([i['type'] for i in items]) | |
| 257 assert not [i for i in types | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
258 if not [True for fetcher in fetchers |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
259 if fetcher.match(i)]] |
| 4 | 260 |
| 15 | 261 for item in items: |
| 4 | 262 |
| 15 | 263 # fix up relative paths |
| 264 dest = item['dest'] | |
| 265 if not os.path.isabs(dest): | |
|
31
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
266 relative_to = self.relative_to |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
267 if not relative_to: |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
268 if isinstance(item['manifest'], basestring): |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
269 relative_to = os.path.dirname(os.path.abspath(item['manifest'])) |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
270 else: |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
271 relative_to = os.getcwd() |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
272 dest = os.path.normpath(os.path.join(relative_to, dest)) |
| 4 | 273 |
| 15 | 274 # fetch the items |
| 275 self(item['url'], destination=dest, type=item['type'], **item['options']) | |
| 0 | 276 |
| 21 | 277 |
| 0 | 278 format_string = "[URL] [destination] [type] <options>" |
| 279 def read_manifests(*manifests): | |
| 15 | 280 """ |
| 281 read some manifests and return the items | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
282 |
| 15 | 283 Format: |
| 284 %s | |
| 285 """ % format_string | |
| 0 | 286 |
| 15 | 287 retval = [] |
| 0 | 288 |
| 15 | 289 for manifest in manifests: |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
290 if isinstance(manifest, basestring): |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
291 assert os.path.exists(manifest), "manifest '%s' not found" % manifest |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
292 f = file(manifest) |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
293 else: |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
294 f = manifest |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
295 |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
296 for line in f.readlines(): |
| 15 | 297 line = line.strip() |
| 298 if line.startswith('#') or not line: | |
| 299 continue | |
| 300 line = line.split() | |
| 301 if len(line) not in (3,4): | |
| 302 raise Exception("Format should be: %s; line %s" % (format_string, line)) | |
| 303 options = {} | |
| 304 if len(line) == 4: | |
| 305 option_string = line.pop().rstrip(',') | |
| 306 try: | |
| 307 options = dict([[j.strip() for j in i.split('=', 1)] | |
| 308 for i in option_string.split(',')]) | |
| 309 except: | |
| 310 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) | |
| 0 | 311 |
| 15 | 312 url, dest, _type = line |
| 313 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) | |
| 314 return retval | |
| 0 | 315 |
| 2 | 316 def main(args=sys.argv[1:]): |
| 0 | 317 |
| 15 | 318 # parse command line options |
| 319 usage = '%prog [options] manifest [manifest] [...]' | |
| 0 | 320 |
| 15 | 321 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): |
| 322 def format_description(self, description): | |
| 323 if description: | |
| 324 return description + '\n' | |
| 325 else: | |
| 326 return '' | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
327 |
| 15 | 328 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) |
| 329 parser.add_option('-o', '--output', | |
| 330 help="output relative to this location vs. the manifest location") | |
| 17 | 331 parser.add_option('-d', '--dest', # XXX unused |
| 15 | 332 action='append', |
| 333 help="output only these destinations") | |
| 334 parser.add_option('-s', '--strict', | |
| 335 action='store_true', default=False, | |
| 336 help="fail on error") | |
| 337 parser.add_option('--list-fetchers', dest='list_fetchers', | |
| 338 action='store_true', default=False, | |
| 339 help='list available fetchers and exit') | |
| 340 options, args = parser.parse_args(args) | |
| 0 | 341 |
| 15 | 342 if options.list_fetchers: |
| 17 | 343 types = set() |
| 344 for fetcher in fetchers: | |
| 345 if fetcher.type in types: | |
| 346 continue # occluded, should probably display separately | |
| 347 print '%s : %s' % (fetcher.type, fetcher.doc()) | |
| 348 types.add(fetcher.type) | |
| 15 | 349 parser.exit() |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
350 |
| 15 | 351 if not args: |
| 17 | 352 # TODO: could read from stdin |
| 15 | 353 parser.print_help() |
| 354 parser.exit() | |
| 0 | 355 |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
356 # sanity check |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
357 assert not [i for i in args if not os.path.exists(i)] |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
358 |
| 15 | 359 items = read_manifests(*args) |
| 32 | 360 fetch = Fetch(fetchers, relative_to=options.output, strict=options.strict) |
| 0 | 361 |
| 15 | 362 # download the files |
| 363 fetch.fetch(*items) | |
| 0 | 364 |
| 365 if __name__ == '__main__': | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
366 main() |
| 0 | 367 |