Mercurial > hg > fetch

comparison fetch.py @ 37:f30fe9183e64

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
remove clobber functionality for now; this should live in the master Fetch class so that resources arent multiply clobbered
author Jeff Hammel <jhammel@mozilla.com>
date Tue, 15 Nov 2011 12:31:35 -0800
parents d3558b202acb
children 645d02834042
comparison
equal deleted inserted replaced
36:d3558b202acb 37:f30fe9183e64
23 23
24 @classmethod 24 @classmethod
25 def match(cls, _type): 25 def match(cls, _type):
26 return _type == cls.type 26 return _type == cls.type
27 27
28 def __init__(self, url, clobber=True): 28 def __init__(self, url):
29 self.subpath = None 29 self.subpath = None
30 if '#' in url: 30 if '#' in url:
31 url, self.subpath = url.rsplit('#') 31 url, self.subpath = url.rsplit('#')
32 if self.subpath: 32 if self.subpath:
33 self.subpath = self.subpath.split('/') 33 self.subpath = self.subpath.split('/')
34 self.url = url 34 self.url = url
35 self._clobber = clobber
36 35
37 def __call__(self, dest): 36 def __call__(self, dest):
38 raise NotImplementedError("Should be called by implementing class") 37 raise NotImplementedError("Should be called by implementing class")
39
40 def clobber(self, dest):
41 """clobbers if self._clobber is set"""
42 if self._clobber and os.path.exists(dest):
43 if os.path.isfile(dest):
44 os.remove(dest)
45 else:
46 shutil.rmtree(dest)
47 38
48 @classmethod 39 @classmethod
49 def doc(cls): 40 def doc(cls):
50 """return docstring for the instance""" 41 """return docstring for the instance"""
51 retval = getattr(cls, '__doc__', '').strip() 42 retval = getattr(cls, '__doc__', '').strip()
57 import urllib2 48 import urllib2
58 from StringIO import StringIO 49 from StringIO import StringIO
59 50
60 class FileFetcher(Fetcher): 51 class FileFetcher(Fetcher):
61 """fetch a single file""" 52 """fetch a single file"""
62 # Note: subpath and clobber for single files are ignored 53 # Note: subpath for single files is ignored
63 54
64 type = 'file' 55 type = 'file'
65 56
66 @classmethod 57 @classmethod
67 def download(cls, url): 58 def download(cls, url):
81 """fetch and extract a tarball""" 72 """fetch and extract a tarball"""
82 73
83 type = 'tar' 74 type = 'tar'
84 75
85 def __call__(self, dest): 76 def __call__(self, dest):
86 self.clobber(dest)
87 if os.path.exists(dest): 77 if os.path.exists(dest):
88 assert os.path.isdir(dest), "Destination must be a directory" 78 assert os.path.isdir(dest), "Destination must be a directory"
89 else: 79 else:
90 os.makedirs(dest) 80 os.makedirs(dest)
91 if self.subpath: 81 if self.subpath:
123 self.prog = self.type # name of app program 113 self.prog = self.type # name of app program
124 self.vcs_dir = '.' + self.type # subdirectory for version control 114 self.vcs_dir = '.' + self.type # subdirectory for version control
125 115
126 def __call__(self, dest): 116 def __call__(self, dest):
127 117
128 self.clobber(dest)
129 if self.subpath or self.export: 118 if self.subpath or self.export:
130 # can only export with a subpath 119 # can only export with a subpath
131 self.export(dest, subpath=self.subpath) 120 self.export(dest, subpath=self.subpath)
132 return 121 return
133 122
145 self.clone(tmpdir) 134 self.clone(tmpdir)
146 path = tmpdir 135 path = tmpdir
147 if self.subpath: 136 if self.subpath:
148 path = os.path.join([tmpdir] + self.subpath) 137 path = os.path.join([tmpdir] + self.subpath)
149 assert os.path.exists(path), "subpath %s of %s not found" % (os.path.sep.join(self.subpath), self.url) 138 assert os.path.exists(path), "subpath %s of %s not found" % (os.path.sep.join(self.subpath), self.url)
150 self.clobber(dest)
151 if os.path.isdir(path): 139 if os.path.isdir(path):
152 if os.path.exists(dest): 140 if os.path.exists(dest):
153 assert os.path.isdir(dest), "source is a directory; destination is a file" 141 assert os.path.isdir(dest), "source is a directory; destination is a file"
154 else: 142 else:
155 os.makedirs(dest) 143 os.makedirs(dest)
231 219
232 __all__ += [i.__name__ for i in fetchers] 220 __all__ += [i.__name__ for i in fetchers]
233 221
234 class Fetch(object): 222 class Fetch(object):
235 223
236 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True): 224 def __init__(self, fetchers=fetchers[:], relative_to=None, strict=True, clobber=True):
237 self.fetchers = fetchers 225 self.fetchers = fetchers
238 self.relative_to = relative_to 226 self.relative_to = relative_to
239 self.strict = strict 227 self.strict = strict
228 self._clobber = clobber
240 229
241 def fetcher(self, _type): 230 def fetcher(self, _type):
242 """find the fetcher for the appropriate type""" 231 """find the fetcher for the appropriate type"""
243 for fetcher in fetchers: 232 for fetcher in fetchers:
244 if fetcher.match(_type): 233 if fetcher.match(_type):
247 def __call__(self, url, destination, type, **options): 236 def __call__(self, url, destination, type, **options):
248 fetcher = self.fetcher(type) 237 fetcher = self.fetcher(type)
249 assert fetcher is not None, "No fetcher found for type '%s'" % type 238 assert fetcher is not None, "No fetcher found for type '%s'" % type
250 fetcher = fetcher(url, **options) 239 fetcher = fetcher(url, **options)
251 fetcher(destination) 240 fetcher(destination)
241
242 def clobber(self, dest):
243 """clobbers if self._clobber is set"""
244 if self._clobber and os.path.exists(dest):
245 if os.path.isfile(dest):
246 os.remove(dest)
247 else:
248 shutil.rmtree(dest)
252 249
253 def fetch(self, *items): 250 def fetch(self, *items):
254 251
255 if self.strict: 252 if self.strict:
256 # ensure all the required fetchers are available 253 # ensure all the required fetchers are available