Mercurial > hg > fetch
annotate fetch.py @ 52:c9f2311dedea
inherit from the right ABC
author | Jeff Hammel <jhammel@mozilla.com> |
---|---|
date | Tue, 15 Nov 2011 15:14:45 -0800 |
parents | 45f7a0f80e6a |
children | 81fe0523a078 |
rev | line source |
---|---|
0 | 1 #!/usr/bin/env python |
2 | |
3 """ | |
4 fetch stuff from the interwebs | |
5 """ | |
6 | |
44 | 7 import optparse |
0 | 8 import os |
23 | 9 import shutil |
0 | 10 import sys |
44 | 11 import tempfile |
0 | 12 |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
13 __all__ = ['Fetcher', 'Fetch', 'main'] |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
14 |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
15 def which(executable, path=os.environ['PATH']): |
15 | 16 """python equivalent of which; should really be in the stdlib""" |
17 dirs = path.split(os.pathsep) | |
18 for dir in dirs: | |
19 if os.path.isfile(os.path.join(dir, executable)): | |
20 return os.path.join(dir, executable) | |
7 | 21 |
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
22 def copytree(src, dst): |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
23 """ |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
24 replacement for shutil.copytree because of this nonsense from help(shutil.copytree): |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
25 "The destination directory must not already exist." |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
26 """ |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
27 |
47 | 28 assert os.path.exists(src), "'%s' does not exist" % src |
29 | |
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
30 # if its a file, just copy it |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
31 if os.path.isfile(src): |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
32 shutil.copy2(src, dst) |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
33 return |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
34 |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
35 # otherwise a directory |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
36 assert os.path.isdir(src) |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
37 if os.path.exists(dst): |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
38 assert os.path.isdir(dst), "%s is a file, %s is a directory" % (src, dst) |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
39 else: |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
40 os.makedirs(dst) |
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
41 |
47 | 42 src = os.path.realpath(src) |
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
43 for dirpath, dirnames, filenames in os.walk(src): |
47 | 44 for d in dirnames: |
48 | 45 path = os.path.join(dirpath, d) |
46 _dst = os.path.join(dst, os.path.relpath(path, src)) # XXX depends on python 2.5 relpath | |
47 if os.path.exists(_dst): | |
48 assert os.path.isdir(_dst), "%s is a file, %s is a directory" % (src, dst) | |
49 else: | |
50 os.makedirs(_dst) | |
47 | 51 for f in filenames: |
52 path = os.path.join(dirpath, f) | |
48 | 53 _dst = os.path.join(dst, os.path.relpath(path, src)) # XXX depends on python 2.5 relpath |
54 shutil.copy2(path, _dst) | |
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
55 |
0 | 56 class Fetcher(object): |
15 | 57 """abstract base class for resource fetchers""" |
0 | 58 |
15 | 59 @classmethod |
60 def match(cls, _type): | |
61 return _type == cls.type | |
0 | 62 |
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
63 def __init__(self, url): |
17 | 64 self.subpath = None |
65 if '#' in url: | |
66 url, self.subpath = url.rsplit('#') | |
25 | 67 if self.subpath: |
68 self.subpath = self.subpath.split('/') | |
15 | 69 self.url = url |
0 | 70 |
15 | 71 def __call__(self, dest): |
17 | 72 raise NotImplementedError("Should be called by implementing class") |
73 | |
74 @classmethod | |
75 def doc(cls): | |
76 """return docstring for the instance""" | |
77 retval = getattr(cls, '__doc__', '').strip() | |
78 return ' '.join(retval.split()) | |
0 | 79 |
7 | 80 ### standard dispatchers - always available |
0 | 81 |
7 | 82 import tarfile |
0 | 83 import urllib2 |
7 | 84 from StringIO import StringIO |
0 | 85 |
5 | 86 class FileFetcher(Fetcher): |
15 | 87 """fetch a single file""" |
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
88 # Note: subpath for single files is ignored |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
89 |
15 | 90 type = 'file' |
0 | 91 |
15 | 92 @classmethod |
93 def download(cls, url): | |
94 return urllib2.urlopen(url).read() | |
0 | 95 |
15 | 96 def __call__(self, dest): |
25 | 97 |
15 | 98 if os.path.isdir(dest): |
99 filename = self.url.rsplit('/', 1)[-1] | |
100 dest = os.path.join(dest, filename) | |
101 f = file(dest, 'w') | |
102 f.write(self.download(self.url)) | |
103 f.close() | |
0 | 104 |
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
105 |
5 | 106 class TarballFetcher(FileFetcher): |
15 | 107 """fetch and extract a tarball""" |
0 | 108 |
15 | 109 type = 'tar' |
0 | 110 |
15 | 111 def __call__(self, dest): |
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
112 if os.path.exists(dest): |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
113 assert os.path.isdir(dest), "Destination must be a directory" |
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
114 else: |
35 | 115 os.makedirs(dest) |
15 | 116 buffer = StringIO() |
117 buffer.write(self.download(self.url)) | |
118 buffer.seek(0) | |
119 tf = tarfile.open(mode='r', fileobj=buffer) | |
38 | 120 members = tf.getmembers() |
39
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
121 if self.subpath: |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
122 |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
123 # build list of files to extract |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
124 _members = [] |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
125 |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
126 toppath = None |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
127 for member in members: |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
128 split = member.name.split(os.path.sep) |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
129 if toppath: |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
130 # ensure that for subpaths that only one top level directory exists |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
131 # XXX needed? |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
132 assert toppath == split[0], "Multiple top-level archives found" |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
133 else: |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
134 toppath = split[0] |
40
e103ae19c2a0
get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
135 if split and split[1:len(self.subpath)+1] == self.subpath: |
e103ae19c2a0
get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
136 member.name = os.path.sep.join(split[1:]) |
e103ae19c2a0
get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
137 _members.append(member) |
39
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
138 |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
139 members = _members |
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
140 |
38 | 141 for member in members: |
142 tf.extract(member, dest) | |
7 | 143 |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
144 fetchers = [FileFetcher, TarballFetcher] |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
145 |
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
146 ### VCS fetchers |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
147 |
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
148 import subprocess |
19 | 149 try: |
150 from subprocess import check_call as call | |
151 except ImportErorr: | |
152 raise # we need check_call, kinda | |
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
153 |
17 | 154 class VCSFetcher(Fetcher): |
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
155 |
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
156 command = None # name of the VCS command (currently unused) |
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
157 |
17 | 158 def __init__(self, url, export=True): |
159 """ | |
160 - export : whether to strip the versioning information | |
161 """ | |
162 Fetcher.__init__(self, url) | |
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
163 self._export = export |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
164 self.prog = self.type # name of app program |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
165 self.vcs_dir = '.' + self.type # subdirectory for version control |
17 | 166 |
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
167 def call(*args, **kwargs): |
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
168 assert command is not None, "Abstract base class" |
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
169 call([self.command] + list(args), **kwargs) |
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
170 |
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
171 def __call__(self, dest): |
25 | 172 |
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
173 if self.subpath or self._export: |
25 | 174 # can only export with a subpath |
43 | 175 self.export(dest) |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
176 return |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
177 |
25 | 178 if os.path.exists(dest): |
179 assert os.path.isdir(dest) | |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
180 else: |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
181 self.clone(dest) |
25 | 182 |
183 def export(self, dest): | |
184 """ | |
185 export a clone of the directory | |
186 """ | |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
187 dest = os.path.abspath(dest) |
45 | 188 tmpdir = tempfile.mkdtemp() |
25 | 189 self.clone(tmpdir) |
47 | 190 if self.vcs_dir: |
191 shutil.rmtree(os.path.join(tmpdir, self.vcs_dir)) | |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
192 path = tmpdir |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
193 if self.subpath: |
51
45f7a0f80e6a
correctly call os.path.join
Jeff Hammel <jhammel@mozilla.com>
parents:
48
diff
changeset
|
194 path = os.path.join(*([tmpdir] + self.subpath)) |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
195 assert os.path.exists(path), "subpath %s of %s not found" % (os.path.sep.join(self.subpath), self.url) |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
196 if os.path.isdir(path): |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
197 if os.path.exists(dest): |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
198 assert os.path.isdir(dest), "source is a directory; destination is a file" |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
199 else: |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
200 os.makedirs(dest) |
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
201 copytree(path, dest) |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
202 else: |
27 | 203 if not os.path.exists(dest): |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
204 directory, filename = os.path.split(dest) |
27 | 205 if os.path.exists(directory): |
206 assert os.path.isdir(directory), "%s is not a directory" % directory | |
207 else: | |
208 os.makedirs(directory) | |
209 shutil.copy(path, dest) | |
25 | 210 shutil.rmtree(tmpdir) |
211 | |
212 def clone(self, dest): | |
213 """ | |
214 clones into a directory | |
215 """ | |
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
216 raise NotImplementedError("Abstract base class") |
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
217 |
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
218 def update(self, dest): |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
219 """ |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
220 updates a checkout |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
221 """ |
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
222 raise NotImplementedError("Abstract base class") |
25 | 223 |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
224 def versioned(self, directory): |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
225 return os.path.exists(os.path.join(directory, self.vcs_dir)) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
226 |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
227 |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
228 if which('hg'): |
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
229 |
17 | 230 class HgFetcher(VCSFetcher): |
15 | 231 """checkout a mercurial repository""" |
232 type = 'hg' | |
0 | 233 |
19 | 234 def __init__(self, url, export=True): |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
235 VCSFetcher.__init__(self, url, export=export) |
19 | 236 self.hg = which('hg') |
25 | 237 assert self.hg, "'hg' command not found" |
19 | 238 |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
239 def clone(self, dest): |
23 | 240 if os.path.exists(dest): |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
241 assert os.path.isdir(dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
242 call([self.hg, 'clone', self.url, dest]) |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
243 |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
244 def update(self, dest): |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
245 assert os.path.versioned(dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
246 assert os.path.exists(dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
247 call([self.hg, 'pull', self.url], cwd=dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
248 call([self.hg, 'update', '-C'], cwd=dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
249 |
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
250 |
15 | 251 fetchers.append(HgFetcher) |
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
252 |
15 | 253 if which('git'): |
17 | 254 |
52
c9f2311dedea
inherit from the right ABC
Jeff Hammel <jhammel@mozilla.com>
parents:
51
diff
changeset
|
255 class GitFetcher(VCSFetcher): |
15 | 256 """checkout a git repository""" |
257 type = 'git' | |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
258 |
20 | 259 def __init__(self, url, export=True): |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
260 VCSFetcher.__init__(self, url, export=export) |
23 | 261 self.git = which('git') |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
262 assert self.git, "'git' command not found" |
23 | 263 |
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
264 def update(self, dest): |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
265 assert os.path.isdir(dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
266 assert os.path.versioned(dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
267 call([self.git, 'pull', self.url], cwd=dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
268 call([self.git, 'update'], cwd=dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
269 |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
270 def clone(self, dest): |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
271 if not os.path.exists(dest): |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
272 os.makedirs(dest) |
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
273 call([self.git, 'clone', self.url, dest]) |
20 | 274 |
275 fetchers.append(GitFetcher) | |
17 | 276 |
16 | 277 __all__ += [i.__name__ for i in fetchers] |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
278 |
0 | 279 class Fetch(object): |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
280 |
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
281 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True, clobber=True): |
15 | 282 self.fetchers = fetchers |
283 self.relative_to = relative_to | |
284 self.strict = strict | |
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
285 self._clobber = clobber |
0 | 286 |
15 | 287 def fetcher(self, _type): |
288 """find the fetcher for the appropriate type""" | |
289 for fetcher in fetchers: | |
290 if fetcher.match(_type): | |
291 return fetcher | |
0 | 292 |
15 | 293 def __call__(self, url, destination, type, **options): |
294 fetcher = self.fetcher(type) | |
295 assert fetcher is not None, "No fetcher found for type '%s'" % type | |
296 fetcher = fetcher(url, **options) | |
297 fetcher(destination) | |
2 | 298 |
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
299 def clobber(self, dest): |
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
300 """clobbers if self._clobber is set""" |
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
301 if self._clobber and os.path.exists(dest): |
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
302 if os.path.isfile(dest): |
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
303 os.remove(dest) |
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
304 else: |
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
305 shutil.rmtree(dest) |
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
306 |
15 | 307 def fetch(self, *items): |
2 | 308 |
15 | 309 if self.strict: |
310 # ensure all the required fetchers are available | |
311 types = set([i['type'] for i in items]) | |
312 assert not [i for i in types | |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
313 if not [True for fetcher in fetchers |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
314 if fetcher.match(i)]] |
4 | 315 |
15 | 316 for item in items: |
4 | 317 |
15 | 318 # fix up relative paths |
319 dest = item['dest'] | |
320 if not os.path.isabs(dest): | |
31
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
321 relative_to = self.relative_to |
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
322 if not relative_to: |
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
323 if isinstance(item['manifest'], basestring): |
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
324 relative_to = os.path.dirname(os.path.abspath(item['manifest'])) |
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
325 else: |
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
326 relative_to = os.getcwd() |
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
327 dest = os.path.normpath(os.path.join(relative_to, dest)) |
4 | 328 |
15 | 329 # fetch the items |
330 self(item['url'], destination=dest, type=item['type'], **item['options']) | |
0 | 331 |
21 | 332 |
0 | 333 format_string = "[URL] [destination] [type] <options>" |
334 def read_manifests(*manifests): | |
15 | 335 """ |
336 read some manifests and return the items | |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
337 |
15 | 338 Format: |
339 %s | |
340 """ % format_string | |
0 | 341 |
15 | 342 retval = [] |
0 | 343 |
15 | 344 for manifest in manifests: |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
345 if isinstance(manifest, basestring): |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
346 assert os.path.exists(manifest), "manifest '%s' not found" % manifest |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
347 f = file(manifest) |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
348 else: |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
349 f = manifest |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
350 |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
351 for line in f.readlines(): |
15 | 352 line = line.strip() |
353 if line.startswith('#') or not line: | |
354 continue | |
355 line = line.split() | |
356 if len(line) not in (3,4): | |
357 raise Exception("Format should be: %s; line %s" % (format_string, line)) | |
358 options = {} | |
359 if len(line) == 4: | |
360 option_string = line.pop().rstrip(',') | |
361 try: | |
362 options = dict([[j.strip() for j in i.split('=', 1)] | |
363 for i in option_string.split(',')]) | |
364 except: | |
365 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) | |
0 | 366 |
15 | 367 url, dest, _type = line |
368 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) | |
369 return retval | |
0 | 370 |
2 | 371 def main(args=sys.argv[1:]): |
0 | 372 |
15 | 373 # parse command line options |
374 usage = '%prog [options] manifest [manifest] [...]' | |
0 | 375 |
15 | 376 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): |
377 def format_description(self, description): | |
378 if description: | |
379 return description + '\n' | |
380 else: | |
381 return '' | |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
382 |
15 | 383 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) |
384 parser.add_option('-o', '--output', | |
385 help="output relative to this location vs. the manifest location") | |
17 | 386 parser.add_option('-d', '--dest', # XXX unused |
15 | 387 action='append', |
388 help="output only these destinations") | |
389 parser.add_option('-s', '--strict', | |
390 action='store_true', default=False, | |
391 help="fail on error") | |
392 parser.add_option('--list-fetchers', dest='list_fetchers', | |
393 action='store_true', default=False, | |
394 help='list available fetchers and exit') | |
395 options, args = parser.parse_args(args) | |
0 | 396 |
15 | 397 if options.list_fetchers: |
17 | 398 types = set() |
399 for fetcher in fetchers: | |
400 if fetcher.type in types: | |
401 continue # occluded, should probably display separately | |
402 print '%s : %s' % (fetcher.type, fetcher.doc()) | |
403 types.add(fetcher.type) | |
15 | 404 parser.exit() |
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
405 |
15 | 406 if not args: |
17 | 407 # TODO: could read from stdin |
15 | 408 parser.print_help() |
409 parser.exit() | |
0 | 410 |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
411 # sanity check |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
412 assert not [i for i in args if not os.path.exists(i)] |
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
413 |
15 | 414 items = read_manifests(*args) |
32 | 415 fetch = Fetch(fetchers, relative_to=options.output, strict=options.strict) |
0 | 416 |
15 | 417 # download the files |
418 fetch.fetch(*items) | |
0 | 419 |
420 if __name__ == '__main__': | |
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
421 main() |
0 | 422 |