Mercurial > hg > fetch
annotate fetch.py @ 47:72d461e2ccbd
more stubbing of copytree
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Tue, 15 Nov 2011 15:00:10 -0800 |
| parents | fb05b7616bd8 |
| children | bedef9761af9 |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 | |
| 3 """ | |
| 4 fetch stuff from the interwebs | |
| 5 """ | |
| 6 | |
| 44 | 7 import optparse |
| 0 | 8 import os |
| 23 | 9 import shutil |
| 0 | 10 import sys |
| 44 | 11 import tempfile |
| 0 | 12 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
13 __all__ = ['Fetcher', 'Fetch', 'main'] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
14 |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
15 def which(executable, path=os.environ['PATH']): |
| 15 | 16 """python equivalent of which; should really be in the stdlib""" |
| 17 dirs = path.split(os.pathsep) | |
| 18 for dir in dirs: | |
| 19 if os.path.isfile(os.path.join(dir, executable)): | |
| 20 return os.path.join(dir, executable) | |
| 7 | 21 |
|
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
22 def copytree(src, dst): |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
23 """ |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
24 replacement for shutil.copytree because of this nonsense from help(shutil.copytree): |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
25 "The destination directory must not already exist." |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
26 """ |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
27 |
| 47 | 28 assert os.path.exists(src), "'%s' does not exist" % src |
| 29 | |
|
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
30 # if its a file, just copy it |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
31 if os.path.isfile(src): |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
32 shutil.copy2(src, dst) |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
33 return |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
34 |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
35 # otherwise a directory |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
36 assert os.path.isdir(src) |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
37 if os.path.exists(dst): |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
38 assert os.path.isdir(dst), "%s is a file, %s is a directory" % (src, dst) |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
39 else: |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
40 os.makedirs(dst) |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
41 |
| 47 | 42 src = os.path.realpath(src) |
|
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
43 for dirpath, dirnames, filenames in os.walk(src): |
| 47 | 44 for d in dirnames: |
| 45 pass | |
| 46 for f in filenames: | |
| 47 path = os.path.join(dirpath, f) | |
| 48 _dst = os.path.join(dst, os.path.relpath(path, src)) | |
| 49 import pdb; pdb.set_trace() | |
| 50 # shutil.copy2(path | |
|
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
51 import pdb; pdb.set_trace() |
|
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
52 |
| 0 | 53 class Fetcher(object): |
| 15 | 54 """abstract base class for resource fetchers""" |
| 0 | 55 |
| 15 | 56 @classmethod |
| 57 def match(cls, _type): | |
| 58 return _type == cls.type | |
| 0 | 59 |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
60 def __init__(self, url): |
| 17 | 61 self.subpath = None |
| 62 if '#' in url: | |
| 63 url, self.subpath = url.rsplit('#') | |
| 25 | 64 if self.subpath: |
| 65 self.subpath = self.subpath.split('/') | |
| 15 | 66 self.url = url |
| 0 | 67 |
| 15 | 68 def __call__(self, dest): |
| 17 | 69 raise NotImplementedError("Should be called by implementing class") |
| 70 | |
| 71 @classmethod | |
| 72 def doc(cls): | |
| 73 """return docstring for the instance""" | |
| 74 retval = getattr(cls, '__doc__', '').strip() | |
| 75 return ' '.join(retval.split()) | |
| 0 | 76 |
| 7 | 77 ### standard dispatchers - always available |
| 0 | 78 |
| 7 | 79 import tarfile |
| 0 | 80 import urllib2 |
| 7 | 81 from StringIO import StringIO |
| 0 | 82 |
| 5 | 83 class FileFetcher(Fetcher): |
| 15 | 84 """fetch a single file""" |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
85 # Note: subpath for single files is ignored |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
86 |
| 15 | 87 type = 'file' |
| 0 | 88 |
| 15 | 89 @classmethod |
| 90 def download(cls, url): | |
| 91 return urllib2.urlopen(url).read() | |
| 0 | 92 |
| 15 | 93 def __call__(self, dest): |
| 25 | 94 |
| 15 | 95 if os.path.isdir(dest): |
| 96 filename = self.url.rsplit('/', 1)[-1] | |
| 97 dest = os.path.join(dest, filename) | |
| 98 f = file(dest, 'w') | |
| 99 f.write(self.download(self.url)) | |
| 100 f.close() | |
| 0 | 101 |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
102 |
| 5 | 103 class TarballFetcher(FileFetcher): |
| 15 | 104 """fetch and extract a tarball""" |
| 0 | 105 |
| 15 | 106 type = 'tar' |
| 0 | 107 |
| 15 | 108 def __call__(self, dest): |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
109 if os.path.exists(dest): |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
110 assert os.path.isdir(dest), "Destination must be a directory" |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
111 else: |
| 35 | 112 os.makedirs(dest) |
| 15 | 113 buffer = StringIO() |
| 114 buffer.write(self.download(self.url)) | |
| 115 buffer.seek(0) | |
| 116 tf = tarfile.open(mode='r', fileobj=buffer) | |
| 38 | 117 members = tf.getmembers() |
|
39
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
118 if self.subpath: |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
119 |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
120 # build list of files to extract |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
121 _members = [] |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
122 |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
123 toppath = None |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
124 for member in members: |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
125 split = member.name.split(os.path.sep) |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
126 if toppath: |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
127 # ensure that for subpaths that only one top level directory exists |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
128 # XXX needed? |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
129 assert toppath == split[0], "Multiple top-level archives found" |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
130 else: |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
131 toppath = split[0] |
|
40
e103ae19c2a0
get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
132 if split and split[1:len(self.subpath)+1] == self.subpath: |
|
e103ae19c2a0
get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
133 member.name = os.path.sep.join(split[1:]) |
|
e103ae19c2a0
get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents:
39
diff
changeset
|
134 _members.append(member) |
|
39
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
135 |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
136 members = _members |
|
8addc3712e75
partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents:
38
diff
changeset
|
137 |
| 38 | 138 for member in members: |
| 139 tf.extract(member, dest) | |
| 7 | 140 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
141 fetchers = [FileFetcher, TarballFetcher] |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
142 |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
143 ### VCS fetchers |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
144 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
145 import subprocess |
| 19 | 146 try: |
| 147 from subprocess import check_call as call | |
| 148 except ImportErorr: | |
| 149 raise # we need check_call, kinda | |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
150 |
| 17 | 151 class VCSFetcher(Fetcher): |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
152 |
|
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
153 command = None # name of the VCS command (currently unused) |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
154 |
| 17 | 155 def __init__(self, url, export=True): |
| 156 """ | |
| 157 - export : whether to strip the versioning information | |
| 158 """ | |
| 159 Fetcher.__init__(self, url) | |
|
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
160 self._export = export |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
161 self.prog = self.type # name of app program |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
162 self.vcs_dir = '.' + self.type # subdirectory for version control |
| 17 | 163 |
|
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
164 def call(*args, **kwargs): |
|
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
165 assert command is not None, "Abstract base class" |
|
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
166 call([self.command] + list(args), **kwargs) |
|
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
167 |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
168 def __call__(self, dest): |
| 25 | 169 |
|
42
6e978ddf5135
self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents:
40
diff
changeset
|
170 if self.subpath or self._export: |
| 25 | 171 # can only export with a subpath |
| 43 | 172 self.export(dest) |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
173 return |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
174 |
| 25 | 175 if os.path.exists(dest): |
| 176 assert os.path.isdir(dest) | |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
177 else: |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
178 self.clone(dest) |
| 25 | 179 |
| 180 def export(self, dest): | |
| 181 """ | |
| 182 export a clone of the directory | |
| 183 """ | |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
184 dest = os.path.abspath(dest) |
| 45 | 185 tmpdir = tempfile.mkdtemp() |
| 25 | 186 self.clone(tmpdir) |
| 47 | 187 if self.vcs_dir: |
| 188 shutil.rmtree(os.path.join(tmpdir, self.vcs_dir)) | |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
189 path = tmpdir |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
190 if self.subpath: |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
191 path = os.path.join([tmpdir] + self.subpath) |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
192 assert os.path.exists(path), "subpath %s of %s not found" % (os.path.sep.join(self.subpath), self.url) |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
193 if os.path.isdir(path): |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
194 if os.path.exists(dest): |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
195 assert os.path.isdir(dest), "source is a directory; destination is a file" |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
196 else: |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
197 os.makedirs(dest) |
|
46
fb05b7616bd8
begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents:
45
diff
changeset
|
198 copytree(path, dest) |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
199 else: |
| 27 | 200 if not os.path.exists(dest): |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
201 directory, filename = os.path.split(dest) |
| 27 | 202 if os.path.exists(directory): |
| 203 assert os.path.isdir(directory), "%s is not a directory" % directory | |
| 204 else: | |
| 205 os.makedirs(directory) | |
| 206 shutil.copy(path, dest) | |
| 25 | 207 shutil.rmtree(tmpdir) |
| 208 | |
| 209 def clone(self, dest): | |
| 210 """ | |
| 211 clones into a directory | |
| 212 """ | |
|
24
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
213 raise NotImplementedError("Abstract base class") |
|
b1f65f3bd1bc
pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents:
23
diff
changeset
|
214 |
|
26
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
215 def update(self, dest): |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
216 """ |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
217 updates a checkout |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
218 """ |
|
d495b610046a
more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents:
25
diff
changeset
|
219 raise NotImplementedError("Abstract base class") |
| 25 | 220 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
221 def versioned(self, directory): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
222 return os.path.exists(os.path.join(directory, self.vcs_dir)) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
223 |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
224 |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
225 if which('hg'): |
|
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
226 |
| 17 | 227 class HgFetcher(VCSFetcher): |
| 15 | 228 """checkout a mercurial repository""" |
| 229 type = 'hg' | |
| 0 | 230 |
| 19 | 231 def __init__(self, url, export=True): |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
232 VCSFetcher.__init__(self, url, export=export) |
| 19 | 233 self.hg = which('hg') |
| 25 | 234 assert self.hg, "'hg' command not found" |
| 19 | 235 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
236 def clone(self, dest): |
| 23 | 237 if os.path.exists(dest): |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
238 assert os.path.isdir(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
239 call([self.hg, 'clone', self.url, dest]) |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
240 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
241 def update(self, dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
242 assert os.path.versioned(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
243 assert os.path.exists(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
244 call([self.hg, 'pull', self.url], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
245 call([self.hg, 'update', '-C'], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
246 |
|
11
726c3d288733
* add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
247 |
| 15 | 248 fetchers.append(HgFetcher) |
|
6
86f6f99e421b
add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents:
5
diff
changeset
|
249 |
| 15 | 250 if which('git'): |
| 17 | 251 |
| 15 | 252 class GitFetcher(Fetcher): |
| 253 """checkout a git repository""" | |
| 254 type = 'git' | |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
255 |
| 20 | 256 def __init__(self, url, export=True): |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
257 VCSFetcher.__init__(self, url, export=export) |
| 23 | 258 self.git = which('git') |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
259 assert self.git, "'git' command not found" |
| 23 | 260 |
|
28
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
261 def update(self, dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
262 assert os.path.isdir(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
263 assert os.path.versioned(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
264 call([self.git, 'pull', self.url], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
265 call([self.git, 'update'], cwd=dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
266 |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
267 def clone(self, dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
268 if not os.path.exists(dest): |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
269 os.makedirs(dest) |
|
5ecb6507931b
fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents:
27
diff
changeset
|
270 call([self.git, 'clone', self.url, dest]) |
| 20 | 271 |
| 272 fetchers.append(GitFetcher) | |
| 17 | 273 |
| 16 | 274 __all__ += [i.__name__ for i in fetchers] |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
275 |
| 0 | 276 class Fetch(object): |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
277 |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
278 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True, clobber=True): |
| 15 | 279 self.fetchers = fetchers |
| 280 self.relative_to = relative_to | |
| 281 self.strict = strict | |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
282 self._clobber = clobber |
| 0 | 283 |
| 15 | 284 def fetcher(self, _type): |
| 285 """find the fetcher for the appropriate type""" | |
| 286 for fetcher in fetchers: | |
| 287 if fetcher.match(_type): | |
| 288 return fetcher | |
| 0 | 289 |
| 15 | 290 def __call__(self, url, destination, type, **options): |
| 291 fetcher = self.fetcher(type) | |
| 292 assert fetcher is not None, "No fetcher found for type '%s'" % type | |
| 293 fetcher = fetcher(url, **options) | |
| 294 fetcher(destination) | |
| 2 | 295 |
|
37
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
296 def clobber(self, dest): |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
297 """clobbers if self._clobber is set""" |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
298 if self._clobber and os.path.exists(dest): |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
299 if os.path.isfile(dest): |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
300 os.remove(dest) |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
301 else: |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
302 shutil.rmtree(dest) |
|
f30fe9183e64
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents:
36
diff
changeset
|
303 |
| 15 | 304 def fetch(self, *items): |
| 2 | 305 |
| 15 | 306 if self.strict: |
| 307 # ensure all the required fetchers are available | |
| 308 types = set([i['type'] for i in items]) | |
| 309 assert not [i for i in types | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
310 if not [True for fetcher in fetchers |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
311 if fetcher.match(i)]] |
| 4 | 312 |
| 15 | 313 for item in items: |
| 4 | 314 |
| 15 | 315 # fix up relative paths |
| 316 dest = item['dest'] | |
| 317 if not os.path.isabs(dest): | |
|
31
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
318 relative_to = self.relative_to |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
319 if not relative_to: |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
320 if isinstance(item['manifest'], basestring): |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
321 relative_to = os.path.dirname(os.path.abspath(item['manifest'])) |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
322 else: |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
323 relative_to = os.getcwd() |
|
60e0e932570e
fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents:
29
diff
changeset
|
324 dest = os.path.normpath(os.path.join(relative_to, dest)) |
| 4 | 325 |
| 15 | 326 # fetch the items |
| 327 self(item['url'], destination=dest, type=item['type'], **item['options']) | |
| 0 | 328 |
| 21 | 329 |
| 0 | 330 format_string = "[URL] [destination] [type] <options>" |
| 331 def read_manifests(*manifests): | |
| 15 | 332 """ |
| 333 read some manifests and return the items | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
334 |
| 15 | 335 Format: |
| 336 %s | |
| 337 """ % format_string | |
| 0 | 338 |
| 15 | 339 retval = [] |
| 0 | 340 |
| 15 | 341 for manifest in manifests: |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
342 if isinstance(manifest, basestring): |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
343 assert os.path.exists(manifest), "manifest '%s' not found" % manifest |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
344 f = file(manifest) |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
345 else: |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
346 f = manifest |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
347 |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
348 for line in f.readlines(): |
| 15 | 349 line = line.strip() |
| 350 if line.startswith('#') or not line: | |
| 351 continue | |
| 352 line = line.split() | |
| 353 if len(line) not in (3,4): | |
| 354 raise Exception("Format should be: %s; line %s" % (format_string, line)) | |
| 355 options = {} | |
| 356 if len(line) == 4: | |
| 357 option_string = line.pop().rstrip(',') | |
| 358 try: | |
| 359 options = dict([[j.strip() for j in i.split('=', 1)] | |
| 360 for i in option_string.split(',')]) | |
| 361 except: | |
| 362 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) | |
| 0 | 363 |
| 15 | 364 url, dest, _type = line |
| 365 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) | |
| 366 return retval | |
| 0 | 367 |
| 2 | 368 def main(args=sys.argv[1:]): |
| 0 | 369 |
| 15 | 370 # parse command line options |
| 371 usage = '%prog [options] manifest [manifest] [...]' | |
| 0 | 372 |
| 15 | 373 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): |
| 374 def format_description(self, description): | |
| 375 if description: | |
| 376 return description + '\n' | |
| 377 else: | |
| 378 return '' | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
379 |
| 15 | 380 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) |
| 381 parser.add_option('-o', '--output', | |
| 382 help="output relative to this location vs. the manifest location") | |
| 17 | 383 parser.add_option('-d', '--dest', # XXX unused |
| 15 | 384 action='append', |
| 385 help="output only these destinations") | |
| 386 parser.add_option('-s', '--strict', | |
| 387 action='store_true', default=False, | |
| 388 help="fail on error") | |
| 389 parser.add_option('--list-fetchers', dest='list_fetchers', | |
| 390 action='store_true', default=False, | |
| 391 help='list available fetchers and exit') | |
| 392 options, args = parser.parse_args(args) | |
| 0 | 393 |
| 15 | 394 if options.list_fetchers: |
| 17 | 395 types = set() |
| 396 for fetcher in fetchers: | |
| 397 if fetcher.type in types: | |
| 398 continue # occluded, should probably display separately | |
| 399 print '%s : %s' % (fetcher.type, fetcher.doc()) | |
| 400 types.add(fetcher.type) | |
| 15 | 401 parser.exit() |
|
8
cf00d46b1bfb
pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents:
7
diff
changeset
|
402 |
| 15 | 403 if not args: |
| 17 | 404 # TODO: could read from stdin |
| 15 | 405 parser.print_help() |
| 406 parser.exit() | |
| 0 | 407 |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
408 # sanity check |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
409 assert not [i for i in args if not os.path.exists(i)] |
|
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
410 |
| 15 | 411 items = read_manifests(*args) |
| 32 | 412 fetch = Fetch(fetchers, relative_to=options.output, strict=options.strict) |
| 0 | 413 |
| 15 | 414 # download the files |
| 415 fetch.fetch(*items) | |
| 0 | 416 |
| 417 if __name__ == '__main__': | |
|
29
1c963875e6cd
add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents:
28
diff
changeset
|
418 main() |
| 0 | 419 |