annotate fetch.py @ 52:c9f2311dedea

inherit from the right ABC
author Jeff Hammel <jhammel@mozilla.com>
date Tue, 15 Nov 2011 15:14:45 -0800
parents 45f7a0f80e6a
children 81fe0523a078
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
1 #!/usr/bin/env python
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
2
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
3 """
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
4 fetch stuff from the interwebs
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
5 """
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
6
44
b98c8a8a881b import tempfile
Jeff Hammel <jhammel@mozilla.com>
parents: 43
diff changeset
7 import optparse
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
8 import os
23
12ad9ab11860 stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 21
diff changeset
9 import shutil
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
10 import sys
44
b98c8a8a881b import tempfile
Jeff Hammel <jhammel@mozilla.com>
parents: 43
diff changeset
11 import tempfile
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
12
8
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
13 __all__ = ['Fetcher', 'Fetch', 'main']
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
14
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
15 def which(executable, path=os.environ['PATH']):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
16 """python equivalent of which; should really be in the stdlib"""
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
17 dirs = path.split(os.pathsep)
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
18 for dir in dirs:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
19 if os.path.isfile(os.path.join(dir, executable)):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
20 return os.path.join(dir, executable)
7
4d3d0e1324f5 flush out tarfile
Jeff Hammel <jhammel@mozilla.com>
parents: 6
diff changeset
21
46
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
22 def copytree(src, dst):
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
23 """
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
24 replacement for shutil.copytree because of this nonsense from help(shutil.copytree):
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
25 "The destination directory must not already exist."
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
26 """
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
27
47
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
28 assert os.path.exists(src), "'%s' does not exist" % src
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
29
46
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
30 # if its a file, just copy it
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
31 if os.path.isfile(src):
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
32 shutil.copy2(src, dst)
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
33 return
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
34
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
35 # otherwise a directory
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
36 assert os.path.isdir(src)
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
37 if os.path.exists(dst):
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
38 assert os.path.isdir(dst), "%s is a file, %s is a directory" % (src, dst)
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
39 else:
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
40 os.makedirs(dst)
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
41
47
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
42 src = os.path.realpath(src)
46
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
43 for dirpath, dirnames, filenames in os.walk(src):
47
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
44 for d in dirnames:
48
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
45 path = os.path.join(dirpath, d)
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
46 _dst = os.path.join(dst, os.path.relpath(path, src)) # XXX depends on python 2.5 relpath
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
47 if os.path.exists(_dst):
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
48 assert os.path.isdir(_dst), "%s is a file, %s is a directory" % (src, dst)
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
49 else:
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
50 os.makedirs(_dst)
47
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
51 for f in filenames:
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
52 path = os.path.join(dirpath, f)
48
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
53 _dst = os.path.join(dst, os.path.relpath(path, src)) # XXX depends on python 2.5 relpath
bedef9761af9 finishing cpytree
Jeff Hammel <jhammel@mozilla.com>
parents: 47
diff changeset
54 shutil.copy2(path, _dst)
46
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
55
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
56 class Fetcher(object):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
57 """abstract base class for resource fetchers"""
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
58
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
59 @classmethod
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
60 def match(cls, _type):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
61 return _type == cls.type
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
62
37
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
63 def __init__(self, url):
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
64 self.subpath = None
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
65 if '#' in url:
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
66 url, self.subpath = url.rsplit('#')
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
67 if self.subpath:
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
68 self.subpath = self.subpath.split('/')
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
69 self.url = url
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
70
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
71 def __call__(self, dest):
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
72 raise NotImplementedError("Should be called by implementing class")
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
73
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
74 @classmethod
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
75 def doc(cls):
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
76 """return docstring for the instance"""
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
77 retval = getattr(cls, '__doc__', '').strip()
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
78 return ' '.join(retval.split())
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
79
7
4d3d0e1324f5 flush out tarfile
Jeff Hammel <jhammel@mozilla.com>
parents: 6
diff changeset
80 ### standard dispatchers - always available
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
81
7
4d3d0e1324f5 flush out tarfile
Jeff Hammel <jhammel@mozilla.com>
parents: 6
diff changeset
82 import tarfile
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
83 import urllib2
7
4d3d0e1324f5 flush out tarfile
Jeff Hammel <jhammel@mozilla.com>
parents: 6
diff changeset
84 from StringIO import StringIO
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
85
5
5b78d1411cdd typo; this now works
Jeff Hammel <jhammel@mozilla.com>
parents: 4
diff changeset
86 class FileFetcher(Fetcher):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
87 """fetch a single file"""
37
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
88 # Note: subpath for single files is ignored
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
89
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
90 type = 'file'
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
91
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
92 @classmethod
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
93 def download(cls, url):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
94 return urllib2.urlopen(url).read()
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
95
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
96 def __call__(self, dest):
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
97
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
98 if os.path.isdir(dest):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
99 filename = self.url.rsplit('/', 1)[-1]
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
100 dest = os.path.join(dest, filename)
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
101 f = file(dest, 'w')
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
102 f.write(self.download(self.url))
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
103 f.close()
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
104
6
86f6f99e421b add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents: 5
diff changeset
105
5
5b78d1411cdd typo; this now works
Jeff Hammel <jhammel@mozilla.com>
parents: 4
diff changeset
106 class TarballFetcher(FileFetcher):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
107 """fetch and extract a tarball"""
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
108
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
109 type = 'tar'
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
110
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
111 def __call__(self, dest):
24
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
112 if os.path.exists(dest):
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
113 assert os.path.isdir(dest), "Destination must be a directory"
24
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
114 else:
35
d9b9c4599a3d use correct name
Jeff Hammel <jhammel@mozilla.com>
parents: 34
diff changeset
115 os.makedirs(dest)
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
116 buffer = StringIO()
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
117 buffer.write(self.download(self.url))
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
118 buffer.seek(0)
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
119 tf = tarfile.open(mode='r', fileobj=buffer)
38
645d02834042 extract....correctly?
Jeff Hammel <jhammel@mozilla.com>
parents: 37
diff changeset
120 members = tf.getmembers()
39
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
121 if self.subpath:
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
122
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
123 # build list of files to extract
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
124 _members = []
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
125
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
126 toppath = None
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
127 for member in members:
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
128 split = member.name.split(os.path.sep)
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
129 if toppath:
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
130 # ensure that for subpaths that only one top level directory exists
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
131 # XXX needed?
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
132 assert toppath == split[0], "Multiple top-level archives found"
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
133 else:
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
134 toppath = split[0]
40
e103ae19c2a0 get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents: 39
diff changeset
135 if split and split[1:len(self.subpath)+1] == self.subpath:
e103ae19c2a0 get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents: 39
diff changeset
136 member.name = os.path.sep.join(split[1:])
e103ae19c2a0 get subpaths working for tarballs
Jeff Hammel <jhammel@mozilla.com>
parents: 39
diff changeset
137 _members.append(member)
39
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
138
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
139 members = _members
8addc3712e75 partially and borken support for subpaths
Jeff Hammel <jhammel@mozilla.com>
parents: 38
diff changeset
140
38
645d02834042 extract....correctly?
Jeff Hammel <jhammel@mozilla.com>
parents: 37
diff changeset
141 for member in members:
645d02834042 extract....correctly?
Jeff Hammel <jhammel@mozilla.com>
parents: 37
diff changeset
142 tf.extract(member, dest)
7
4d3d0e1324f5 flush out tarfile
Jeff Hammel <jhammel@mozilla.com>
parents: 6
diff changeset
143
8
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
144 fetchers = [FileFetcher, TarballFetcher]
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
145
24
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
146 ### VCS fetchers
8
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
147
11
726c3d288733 * add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents: 10
diff changeset
148 import subprocess
19
d69041957c0e more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 18
diff changeset
149 try:
d69041957c0e more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 18
diff changeset
150 from subprocess import check_call as call
d69041957c0e more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 18
diff changeset
151 except ImportErorr:
d69041957c0e more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 18
diff changeset
152 raise # we need check_call, kinda
11
726c3d288733 * add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents: 10
diff changeset
153
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
154 class VCSFetcher(Fetcher):
24
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
155
42
6e978ddf5135 self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents: 40
diff changeset
156 command = None # name of the VCS command (currently unused)
24
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
157
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
158 def __init__(self, url, export=True):
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
159 """
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
160 - export : whether to strip the versioning information
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
161 """
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
162 Fetcher.__init__(self, url)
42
6e978ddf5135 self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents: 40
diff changeset
163 self._export = export
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
164 self.prog = self.type # name of app program
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
165 self.vcs_dir = '.' + self.type # subdirectory for version control
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
166
42
6e978ddf5135 self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents: 40
diff changeset
167 def call(*args, **kwargs):
6e978ddf5135 self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents: 40
diff changeset
168 assert command is not None, "Abstract base class"
6e978ddf5135 self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents: 40
diff changeset
169 call([self.command] + list(args), **kwargs)
6e978ddf5135 self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents: 40
diff changeset
170
24
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
171 def __call__(self, dest):
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
172
42
6e978ddf5135 self._export vs self.export, the function
Jeff Hammel <jhammel@mozilla.com>
parents: 40
diff changeset
173 if self.subpath or self._export:
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
174 # can only export with a subpath
43
9e6c0f9725e3 remove extraneous keyword
Jeff Hammel <jhammel@mozilla.com>
parents: 42
diff changeset
175 self.export(dest)
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
176 return
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
177
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
178 if os.path.exists(dest):
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
179 assert os.path.isdir(dest)
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
180 else:
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
181 self.clone(dest)
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
182
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
183 def export(self, dest):
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
184 """
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
185 export a clone of the directory
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
186 """
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
187 dest = os.path.abspath(dest)
45
87c22bbcda2b use correct function name
Jeff Hammel <jhammel@mozilla.com>
parents: 44
diff changeset
188 tmpdir = tempfile.mkdtemp()
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
189 self.clone(tmpdir)
47
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
190 if self.vcs_dir:
72d461e2ccbd more stubbing of copytree
Jeff Hammel <jhammel@mozilla.com>
parents: 46
diff changeset
191 shutil.rmtree(os.path.join(tmpdir, self.vcs_dir))
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
192 path = tmpdir
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
193 if self.subpath:
51
45f7a0f80e6a correctly call os.path.join
Jeff Hammel <jhammel@mozilla.com>
parents: 48
diff changeset
194 path = os.path.join(*([tmpdir] + self.subpath))
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
195 assert os.path.exists(path), "subpath %s of %s not found" % (os.path.sep.join(self.subpath), self.url)
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
196 if os.path.isdir(path):
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
197 if os.path.exists(dest):
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
198 assert os.path.isdir(dest), "source is a directory; destination is a file"
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
199 else:
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
200 os.makedirs(dest)
46
fb05b7616bd8 begin stubbing own copytree function :(
Jeff Hammel <jhammel@mozilla.com>
parents: 45
diff changeset
201 copytree(path, dest)
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
202 else:
27
423b67ff4512 more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 26
diff changeset
203 if not os.path.exists(dest):
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
204 directory, filename = os.path.split(dest)
27
423b67ff4512 more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 26
diff changeset
205 if os.path.exists(directory):
423b67ff4512 more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 26
diff changeset
206 assert os.path.isdir(directory), "%s is not a directory" % directory
423b67ff4512 more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 26
diff changeset
207 else:
423b67ff4512 more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 26
diff changeset
208 os.makedirs(directory)
423b67ff4512 more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 26
diff changeset
209 shutil.copy(path, dest)
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
210 shutil.rmtree(tmpdir)
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
211
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
212 def clone(self, dest):
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
213 """
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
214 clones into a directory
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
215 """
24
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
216 raise NotImplementedError("Abstract base class")
b1f65f3bd1bc pretend to flesh out git fetcher
Jeff Hammel <jhammel@mozilla.com>
parents: 23
diff changeset
217
26
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
218 def update(self, dest):
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
219 """
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
220 updates a checkout
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
221 """
d495b610046a more stubbing; tests work again
Jeff Hammel <jhammel@mozilla.com>
parents: 25
diff changeset
222 raise NotImplementedError("Abstract base class")
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
223
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
224 def versioned(self, directory):
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
225 return os.path.exists(os.path.join(directory, self.vcs_dir))
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
226
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
227
8
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
228 if which('hg'):
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
229
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
230 class HgFetcher(VCSFetcher):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
231 """checkout a mercurial repository"""
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
232 type = 'hg'
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
233
19
d69041957c0e more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 18
diff changeset
234 def __init__(self, url, export=True):
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
235 VCSFetcher.__init__(self, url, export=export)
19
d69041957c0e more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 18
diff changeset
236 self.hg = which('hg')
25
e628ce3ae49f more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 24
diff changeset
237 assert self.hg, "'hg' command not found"
19
d69041957c0e more stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 18
diff changeset
238
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
239 def clone(self, dest):
23
12ad9ab11860 stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 21
diff changeset
240 if os.path.exists(dest):
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
241 assert os.path.isdir(dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
242 call([self.hg, 'clone', self.url, dest])
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
243
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
244 def update(self, dest):
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
245 assert os.path.versioned(dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
246 assert os.path.exists(dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
247 call([self.hg, 'pull', self.url], cwd=dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
248 call([self.hg, 'update', '-C'], cwd=dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
249
11
726c3d288733 * add convenience import in __init__
Jeff Hammel <jhammel@mozilla.com>
parents: 10
diff changeset
250
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
251 fetchers.append(HgFetcher)
6
86f6f99e421b add types for unimplemented dispatchers
Jeff Hammel <jhammel@mozilla.com>
parents: 5
diff changeset
252
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
253 if which('git'):
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
254
52
c9f2311dedea inherit from the right ABC
Jeff Hammel <jhammel@mozilla.com>
parents: 51
diff changeset
255 class GitFetcher(VCSFetcher):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
256 """checkout a git repository"""
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
257 type = 'git'
8
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
258
20
738d84b4de52 start documenting
Jeff Hammel <jhammel@mozilla.com>
parents: 19
diff changeset
259 def __init__(self, url, export=True):
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
260 VCSFetcher.__init__(self, url, export=export)
23
12ad9ab11860 stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 21
diff changeset
261 self.git = which('git')
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
262 assert self.git, "'git' command not found"
23
12ad9ab11860 stubbing
Jeff Hammel <jhammel@mozilla.com>
parents: 21
diff changeset
263
28
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
264 def update(self, dest):
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
265 assert os.path.isdir(dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
266 assert os.path.versioned(dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
267 call([self.git, 'pull', self.url], cwd=dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
268 call([self.git, 'update'], cwd=dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
269
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
270 def clone(self, dest):
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
271 if not os.path.exists(dest):
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
272 os.makedirs(dest)
5ecb6507931b fix vcs fetchers to almost follow a pattern
Jeff Hammel <jhammel@mozilla.com>
parents: 27
diff changeset
273 call([self.git, 'clone', self.url, dest])
20
738d84b4de52 start documenting
Jeff Hammel <jhammel@mozilla.com>
parents: 19
diff changeset
274
738d84b4de52 start documenting
Jeff Hammel <jhammel@mozilla.com>
parents: 19
diff changeset
275 fetchers.append(GitFetcher)
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
276
16
c77d29a10e08 get it working
Jeff Hammel <jhammel@mozilla.com>
parents: 15
diff changeset
277 __all__ += [i.__name__ for i in fetchers]
8
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
278
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
279 class Fetch(object):
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
280
37
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
281 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True, clobber=True):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
282 self.fetchers = fetchers
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
283 self.relative_to = relative_to
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
284 self.strict = strict
37
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
285 self._clobber = clobber
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
286
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
287 def fetcher(self, _type):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
288 """find the fetcher for the appropriate type"""
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
289 for fetcher in fetchers:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
290 if fetcher.match(_type):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
291 return fetcher
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
292
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
293 def __call__(self, url, destination, type, **options):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
294 fetcher = self.fetcher(type)
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
295 assert fetcher is not None, "No fetcher found for type '%s'" % type
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
296 fetcher = fetcher(url, **options)
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
297 fetcher(destination)
2
0c344ed749ac fix typos
Jeff Hammel <jhammel@mozilla.com>
parents: 0
diff changeset
298
37
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
299 def clobber(self, dest):
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
300 """clobbers if self._clobber is set"""
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
301 if self._clobber and os.path.exists(dest):
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
302 if os.path.isfile(dest):
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
303 os.remove(dest)
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
304 else:
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
305 shutil.rmtree(dest)
f30fe9183e64 remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
Jeff Hammel <jhammel@mozilla.com>
parents: 36
diff changeset
306
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
307 def fetch(self, *items):
2
0c344ed749ac fix typos
Jeff Hammel <jhammel@mozilla.com>
parents: 0
diff changeset
308
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
309 if self.strict:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
310 # ensure all the required fetchers are available
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
311 types = set([i['type'] for i in items])
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
312 assert not [i for i in types
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
313 if not [True for fetcher in fetchers
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
314 if fetcher.match(i)]]
4
a984091d681a fix up destination
Jeff Hammel <jhammel@mozilla.com>
parents: 3
diff changeset
315
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
316 for item in items:
4
a984091d681a fix up destination
Jeff Hammel <jhammel@mozilla.com>
parents: 3
diff changeset
317
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
318 # fix up relative paths
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
319 dest = item['dest']
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
320 if not os.path.isabs(dest):
31
60e0e932570e fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents: 29
diff changeset
321 relative_to = self.relative_to
60e0e932570e fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents: 29
diff changeset
322 if not relative_to:
60e0e932570e fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents: 29
diff changeset
323 if isinstance(item['manifest'], basestring):
60e0e932570e fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents: 29
diff changeset
324 relative_to = os.path.dirname(os.path.abspath(item['manifest']))
60e0e932570e fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents: 29
diff changeset
325 else:
60e0e932570e fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents: 29
diff changeset
326 relative_to = os.getcwd()
60e0e932570e fix one error but not the one i was aiming for
Jeff Hammel <jhammel@mozilla.com>
parents: 29
diff changeset
327 dest = os.path.normpath(os.path.join(relative_to, dest))
4
a984091d681a fix up destination
Jeff Hammel <jhammel@mozilla.com>
parents: 3
diff changeset
328
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
329 # fetch the items
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
330 self(item['url'], destination=dest, type=item['type'], **item['options'])
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
331
21
0706968f01bb stub testing
Jeff Hammel <jhammel@mozilla.com>
parents: 20
diff changeset
332
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
333 format_string = "[URL] [destination] [type] <options>"
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
334 def read_manifests(*manifests):
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
335 """
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
336 read some manifests and return the items
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
337
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
338 Format:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
339 %s
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
340 """ % format_string
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
341
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
342 retval = []
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
343
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
344 for manifest in manifests:
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
345 if isinstance(manifest, basestring):
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
346 assert os.path.exists(manifest), "manifest '%s' not found" % manifest
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
347 f = file(manifest)
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
348 else:
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
349 f = manifest
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
350
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
351 for line in f.readlines():
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
352 line = line.strip()
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
353 if line.startswith('#') or not line:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
354 continue
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
355 line = line.split()
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
356 if len(line) not in (3,4):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
357 raise Exception("Format should be: %s; line %s" % (format_string, line))
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
358 options = {}
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
359 if len(line) == 4:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
360 option_string = line.pop().rstrip(',')
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
361 try:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
362 options = dict([[j.strip() for j in i.split('=', 1)]
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
363 for i in option_string.split(',')])
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
364 except:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
365 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string)
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
366
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
367 url, dest, _type = line
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
368 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest))
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
369 return retval
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
370
2
0c344ed749ac fix typos
Jeff Hammel <jhammel@mozilla.com>
parents: 0
diff changeset
371 def main(args=sys.argv[1:]):
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
372
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
373 # parse command line options
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
374 usage = '%prog [options] manifest [manifest] [...]'
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
375
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
376 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
377 def format_description(self, description):
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
378 if description:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
379 return description + '\n'
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
380 else:
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
381 return ''
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
382
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
383 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter())
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
384 parser.add_option('-o', '--output',
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
385 help="output relative to this location vs. the manifest location")
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
386 parser.add_option('-d', '--dest', # XXX unused
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
387 action='append',
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
388 help="output only these destinations")
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
389 parser.add_option('-s', '--strict',
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
390 action='store_true', default=False,
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
391 help="fail on error")
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
392 parser.add_option('--list-fetchers', dest='list_fetchers',
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
393 action='store_true', default=False,
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
394 help='list available fetchers and exit')
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
395 options, args = parser.parse_args(args)
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
396
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
397 if options.list_fetchers:
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
398 types = set()
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
399 for fetcher in fetchers:
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
400 if fetcher.type in types:
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
401 continue # occluded, should probably display separately
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
402 print '%s : %s' % (fetcher.type, fetcher.doc())
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
403 types.add(fetcher.type)
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
404 parser.exit()
8
cf00d46b1bfb pretend like we have a pluggable system to start debugging it
Jeff Hammel <jhammel@mozilla.com>
parents: 7
diff changeset
405
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
406 if not args:
17
e2af4bc5159c more polishing
Jeff Hammel <jhammel@mozilla.com>
parents: 16
diff changeset
407 # TODO: could read from stdin
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
408 parser.print_help()
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
409 parser.exit()
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
410
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
411 # sanity check
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
412 assert not [i for i in args if not os.path.exists(i)]
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
413
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
414 items = read_manifests(*args)
32
46d5bd33311d fix relative_to a bit
Jeff Hammel <jhammel@mozilla.com>
parents: 31
diff changeset
415 fetch = Fetch(fetchers, relative_to=options.output, strict=options.strict)
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
416
15
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
417 # download the files
bc7d6763357e clean up spacing
Jeff Hammel <jhammel@mozilla.com>
parents: 13
diff changeset
418 fetch.fetch(*items)
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
419
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
420 if __name__ == '__main__':
29
1c963875e6cd add a test for manifest and fix resulting bugs
Jeff Hammel <jhammel@mozilla.com>
parents: 28
diff changeset
421 main()
0
3497a30190d2 initial commit of fetch, WIP
Jeff Hammel <jhammel@mozilla.com>
parents:
diff changeset
422