Mercurial > hg > fetch

comparison fetch.py @ 15:bc7d6763357e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
clean up spacing
author Jeff Hammel <jhammel@mozilla.com>
date Wed, 09 Nov 2011 16:29:58 -0800
parents 3fee8ecd1af8
children c77d29a10e08
comparison
equal deleted inserted replaced
14:626b027134ea 15:bc7d6763357e
9 import optparse 9 import optparse
10 10
11 __all__ = ['Fetcher', 'Fetch', 'main'] 11 __all__ = ['Fetcher', 'Fetch', 'main']
12 12
13 def which(executable, path=os.environ['PATH']): 13 def which(executable, path=os.environ['PATH']):
14 """python equivalent of which; should really be in the stdlib""" 14 """python equivalent of which; should really be in the stdlib"""
15 # XXX from https://github.com/mozautomation/mozmill/blob/master/mozrunner/mozrunner/utils.py 15 dirs = path.split(os.pathsep)
16 dirs = path.split(os.pathsep) 16 for dir in dirs:
17 for dir in dirs: 17 if os.path.isfile(os.path.join(dir, executable)):
18 if os.path.isfile(os.path.join(dir, executable)): 18 return os.path.join(dir, executable)
19 return os.path.join(dir, executable)
20
21 19
22 class Fetcher(object): 20 class Fetcher(object):
23 """abstract base class for resource fetchers""" 21 """abstract base class for resource fetchers"""
24 22
25 @classmethod 23 @classmethod
26 def match(cls, _type): 24 def match(cls, _type):
27 return _type == cls.type 25 return _type == cls.type
28 26
29 def __init__(self, url): 27 def __init__(self, url):
30 self.url = url 28 self.url = url
31 29
32 def __call__(self, dest): 30 def __call__(self, dest):
33 raise NotImplementedError 31 raise NotImplementedError
34 32
35 ### standard dispatchers - always available 33 ### standard dispatchers - always available
36 34
37 import tarfile 35 import tarfile
38 import urllib2 36 import urllib2
39 from StringIO import StringIO 37 from StringIO import StringIO
40 38
41 class FileFetcher(Fetcher): 39 class FileFetcher(Fetcher):
42 """fetch a single file""" 40 """fetch a single file"""
43 41
44 type = 'file' 42 type = 'file'
45 43
46 @classmethod 44 @classmethod
47 def download(cls, url): 45 def download(cls, url):
48 return urllib2.urlopen(url).read() 46 return urllib2.urlopen(url).read()
49 47
50 def __call__(self, dest): 48 def __call__(self, dest):
51 if os.path.isdir(dest): 49 if os.path.isdir(dest):
52 filename = self.url.rsplit('/', 1)[-1] 50 filename = self.url.rsplit('/', 1)[-1]
53 dest = os.path.join(dest, filename) 51 dest = os.path.join(dest, filename)
54 f = file(dest, 'w') 52 f = file(dest, 'w')
55 f.write(self.download(self.url)) 53 f.write(self.download(self.url))
56 f.close() 54 f.close()
57 55
58 56
59 class TarballFetcher(FileFetcher): 57 class TarballFetcher(FileFetcher):
60 """fetch and extract a tarball""" 58 """fetch and extract a tarball"""
61 59
62 type = 'tar' 60 type = 'tar'
63 61
64 def __call__(self, dest): 62 def __call__(self, dest):
65 assert os.path.isdir(dest) 63 assert os.path.isdir(dest)
66 buffer = StringIO() 64 buffer = StringIO()
67 buffer.write(self.download(self.url)) 65 buffer.write(self.download(self.url))
68 buffer.seek(0) 66 buffer.seek(0)
69 tf = tarfile.open(mode='r', fileobj=buffer) 67 tf = tarfile.open(mode='r', fileobj=buffer)
70 tf.extract(dest) 68 tf.extract(dest)
71 69
72 fetchers = [FileFetcher, TarballFetcher] 70 fetchers = [FileFetcher, TarballFetcher]
73 71
74 ### VCS fetchers using executable 72 ### VCS fetchers using executable
75 73
76 import subprocess 74 import subprocess
77 75
78 if which('hg'): 76 if which('hg'):
79 77
80 class HgFetcher(Fetcher): 78 class HgFetcher(Fetcher):
81 """checkout a mercurial repository""" 79 """checkout a mercurial repository"""
82 type = 'hg' 80 type = 'hg'
83 81
84 def __call__(self, dest): 82 def __call__(self, dest):
85 if os.path.exits(dest): 83 if os.path.exits(dest):
86 assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg')) 84 assert os.path.isdir(dest) and os.path.exists(os.path.join(dest, '.hg'))
87 pass # TODO 85 pass # TODO
88 86
89 fetchers.append(HgFetcher) 87 fetchers.append(HgFetcher)
90 88
91 class GitFetcher(Fetcher): 89 if which('git'):
92 """checkout a git repository""" 90 class GitFetcher(Fetcher):
93 type = 'git' 91 """checkout a git repository"""
94 92 type = 'git'
95 93
96 fetchers = dict([(i.__name__, i) for i in fetchers]) 94 fetchers = dict([(i.__name__, i) for i in fetchers])
97 __all__ += fetchers.keys() 95 __all__ += fetchers.keys()
98 96
99
100 class Fetch(object): 97 class Fetch(object):
101 98
102 def __init__(self, fetchers, relative_to=None, strict=True): 99 def __init__(self, fetchers, relative_to=None, strict=True):
103 self.fetchers = fetchers 100 self.fetchers = fetchers
104 self.relative_to = relative_to 101 self.relative_to = relative_to
105 self.strict = strict 102 self.strict = strict
106 103
107 def fetcher(self, _type): 104 def fetcher(self, _type):
108 """find the fetcher for the appropriate type""" 105 """find the fetcher for the appropriate type"""
109 for fetcher in fetchers: 106 for fetcher in fetchers:
110 if fetcher.match(_type): 107 if fetcher.match(_type):
111 return fetcher 108 return fetcher
112 109
113 def __call__(self, url, destination, type, **options): 110 def __call__(self, url, destination, type, **options):
114 fetcher = self.fetcher(type) 111 fetcher = self.fetcher(type)
115 assert fetcher is not None 112 assert fetcher is not None, "No fetcher found for type '%s'" % type
116 fetcher = fetcher(url, **options) 113 fetcher = fetcher(url, **options)
117 fetcher(destination) 114 fetcher(destination)
118 115
119 def fetch(self, *items): 116 def fetch(self, *items):
120 117
121 if self.strict: 118 if self.strict:
122 # ensure all the required fetchers are available 119 # ensure all the required fetchers are available
123 types = set([i['type'] for i in items]) 120 types = set([i['type'] for i in items])
124 assert not [i for i in types 121 assert not [i for i in types
125 if [True for fetcher in fetchers if fetcher.match(i)]] 122 if [True for fetcher in fetchers if fetcher.match(i)]]
126 123
127 for item in items: 124 for item in items:
128 125
129 # fix up relative paths 126 # fix up relative paths
130 dest = item['dest'] 127 dest = item['dest']
131 if not os.path.isabs(dest): 128 if not os.path.isabs(dest):
132 if self.relative_to: 129 relative_to = self.relative_to or os.path.dirname(os.path.abspath(item['manifest']))
133 dest = os.path.join(self.relative_to, dest) 130 dest = os.path.join(relative_to, dest)
134 else: 131
135 dest = os.path.join(os.path.dirname(os.path.abspath(item['manifest'])), dest) 132 # fetch the items
136 133 self(item['url'], destination=dest, type=item['type'], **item['options'])
137 # fetch the items
138 self(item['url'], destination=dest, type=item['type'], **item['options'])
139
140 134
141 format_string = "[URL] [destination] [type] <options>" 135 format_string = "[URL] [destination] [type] <options>"
142 def read_manifests(*manifests): 136 def read_manifests(*manifests):
143 """ 137 """
144 read some manifests and return the items 138 read some manifests and return the items
145 139
146 Format: 140 Format:
147 %s 141 %s
148 """ % format_string 142 """ % format_string
149 143
150 # sanity check 144 # sanity check
151 assert not [i for i in manifests if not os.path.exists(i)] 145 assert not [i for i in manifests if not os.path.exists(i)]
152 146
153 retval = [] 147 retval = []
154 148
155 for manifest in manifests: 149 for manifest in manifests:
156 for line in file(i).readlines(): 150 for line in file(i).readlines():
157 line = line.strip() 151 line = line.strip()
158 if line.startswith('#') or not line: 152 if line.startswith('#') or not line:
159 continue 153 continue
160 line = line.split() 154 line = line.split()
161 if len(line) not in (3,4): 155 if len(line) not in (3,4):
162 raise Exception("Format should be: %s; line %s" % (format_string, line)) 156 raise Exception("Format should be: %s; line %s" % (format_string, line))
163 options = {} 157 options = {}
164 if len(line) == 4: 158 if len(line) == 4:
165 option_string = line.pop().rstrip(',') 159 option_string = line.pop().rstrip(',')
166 try: 160 try:
167 options = dict([[j.strip() for j in i.split('=', 1)] 161 options = dict([[j.strip() for j in i.split('=', 1)]
168 for i in option_string.split(',')]) 162 for i in option_string.split(',')])
169 except: 163 except:
170 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string) 164 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string)
171 165
172 url, dest, _type = line 166 url, dest, _type = line
173 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest)) 167 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest))
174 return retval 168 return retval
175 169
176 def main(args=sys.argv[1:]): 170 def main(args=sys.argv[1:]):
177 171
178 # parse command line options 172 # parse command line options
179 usage = '%prog [options] manifest [manifest] [...]' 173 usage = '%prog [options] manifest [manifest] [...]'
180 174
181 # description formatter 175 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
182 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): 176 def format_description(self, description):
183 def format_description(self, description): 177 if description:
184 if description: 178 return description + '\n'
185 return description + '\n' 179 else:
186 else: 180 return ''
187 return ''
188 181
189 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter()) 182 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter())
190 parser.add_option('-o', '--output', 183 parser.add_option('-o', '--output',
191 help="output relative to this location vs. the manifest location") 184 help="output relative to this location vs. the manifest location")
192 parser.add_option('-d', '--dest', 185 parser.add_option('-d', '--dest',
193 action='append', 186 action='append',
194 help="output only these destinations") 187 help="output only these destinations")
195 parser.add_option('-s', '--strict', 188 parser.add_option('-s', '--strict',
196 action='store_true', default=False, 189 action='store_true', default=False,
197 help="fail on error") 190 help="fail on error")
198 parser.add_option('--list-fetchers', dest='list_fetchers', 191 parser.add_option('--list-fetchers', dest='list_fetchers',
199 action='store_true', default=False, 192 action='store_true', default=False,
200 help='list available fetchers and exit') 193 help='list available fetchers and exit')
201 options, args = parser.parse_args(args) 194 options, args = parser.parse_args(args)
202 195
203 if options.list_fetchers: 196 if options.list_fetchers:
204 for name in sorted(fetchers.keys()): 197 for name in sorted(fetchers.keys()):
205 print name 198 print name
206 parser.exit() 199 parser.exit()
207 200
208 if not args: 201 if not args:
209 parser.print_help() 202 parser.print_help()
210 parser.exit() 203 parser.exit()
211 204
212 items = read_manifests(*args) 205 items = read_manifests(*args)
213 fetch = Fetch(fetchers.values(), strict=options.strict) 206 fetch = Fetch(fetchers.values(), strict=options.strict)
214 207
215 # download the files 208 # download the files
216 fetch.fetch(*items) 209 fetch.fetch(*items)
217 210
218 if __name__ == '__main__': 211 if __name__ == '__main__':
219 main() 212 main()
220 213