|
0
|
1 #!/usr/bin/env python
|
|
|
2
|
|
|
3 """
|
|
|
4 fetch stuff from the interwebs
|
|
|
5 """
|
|
|
6
|
|
|
7 import os
|
|
|
8 import sys
|
|
|
9 import optparse
|
|
|
10
|
|
|
11 class Fetcher(object):
|
|
|
12 """abstract base class for resource fetchers"""
|
|
|
13
|
|
|
14 @classmethod
|
|
|
15 def match(cls, _type):
|
|
|
16 return _type == cls.type
|
|
|
17
|
|
|
18 def __init__(self, url):
|
|
|
19 self.url = url
|
|
|
20
|
|
|
21 def __call__(self, dest):
|
|
|
22 raise NotImplementedError
|
|
|
23
|
|
|
24
|
|
|
25 import urllib2
|
|
|
26
|
|
|
27 class FileFetcher(object):
|
|
|
28 """fetch a single file"""
|
|
|
29
|
|
|
30 type = 'file'
|
|
|
31
|
|
|
32 @classmethod
|
|
|
33 def download(cls, url):
|
|
|
34 return urllib2.urlopen(url).read()
|
|
|
35
|
|
|
36 def __call__(self, dest):
|
|
|
37 if os.path.isdir(dest):
|
|
|
38 filename = url.rsplit('/', 1)[-1]
|
|
|
39 dest = os.path.join(dest, filename)
|
|
|
40 f = file(dest, 'w')
|
|
|
41 f.write(self.download(self.url))
|
|
|
42 f.close()
|
|
|
43
|
|
|
44 class TarballFetcher(object):
|
|
|
45 """fetch and extract a tarball"""
|
|
|
46
|
|
|
47 type = 'tar'
|
|
|
48
|
|
|
49 class HgFetcher(object):
|
|
|
50 """checkout a mercurial repository"""
|
|
|
51
|
|
|
52 class GitFetcher(object):
|
|
|
53 """checkout a git repository"""
|
|
|
54
|
|
|
55 fetchers = [FileFetcher]
|
|
|
56
|
|
|
57 class Fetch(object):
|
|
|
58
|
|
|
59 def __init__(self, fetchers, relative_to=None, strict=True):
|
|
|
60 self.fetchers = fetchers
|
|
|
61 self.relative_to = relative_to
|
|
|
62 self.strict = strict
|
|
|
63
|
|
|
64 def fetcher(self, _type):
|
|
|
65 """find the fetcher for the appropriate type"""
|
|
|
66 for fetcher in fetchers:
|
|
|
67 if fetcher.match(_type):
|
|
|
68 return fetcher
|
|
|
69
|
|
|
70 def __call__(self, url, destination, type, **options):
|
|
|
71 fetcher = self.fetcher(type)
|
|
|
72 assert fetcher is not None
|
|
|
73 fetcher = fetcher(url, **options)
|
|
|
74 fetcher(destination)
|
|
|
75
|
|
|
76 def fetch(self, *items):
|
|
2
|
77
|
|
|
78 if self.strict:
|
|
|
79 # ensure all the required fetchers are available
|
|
|
80 types = set([i['type'] for i in items])
|
|
|
81 assert not [i for i in types
|
|
|
82 if [True for fetcher in fetchers if fetcher.match(i)]]
|
|
|
83
|
|
|
84 # fetch the items
|
|
0
|
85 for item in items:
|
|
|
86 self(item['url'])
|
|
|
87
|
|
|
88 format_string = "[URL] [destination] [type] <options>"
|
|
|
89 def read_manifests(*manifests):
|
|
|
90 """
|
|
|
91 read some manifests and return the items
|
|
|
92
|
|
|
93 Format:
|
|
|
94 %s
|
|
|
95 """ % format_string
|
|
|
96
|
|
|
97 # sanity check
|
|
2
|
98 assert not [i for i in manifests if not os.path.exists(i)]
|
|
0
|
99
|
|
|
100 retval = []
|
|
|
101
|
|
|
102 for manifest in manifests:
|
|
|
103 for line in file(i).readlines():
|
|
|
104 line = line.strip()
|
|
3
|
105 if line.startswith('#') or not line:
|
|
|
106 continue
|
|
0
|
107 line = line.split()
|
|
|
108 if len(line) not in (3,4):
|
|
|
109 raise Exception("Format should be: %s; line %s" % (format_string, line))
|
|
|
110 options = {}
|
|
|
111 if len(line) == 4:
|
|
|
112 option_string = line.pop().rstrip(',')
|
|
|
113 try:
|
|
|
114 options = dict([[j.strip() for j in i.split('=', 1)]
|
|
|
115 for i in option_string.split(',')])
|
|
|
116 except:
|
|
|
117 raise Exception("Options format should be: key=value,key2=value2,...; got %s" % option_string)
|
|
|
118
|
|
|
119 url, dest, _type = line
|
|
|
120 retval.append(dict(url=url, dest=dest, type=_type, options=options, manifest=manifest))
|
|
|
121 return retval
|
|
|
122
|
|
2
|
123 def main(args=sys.argv[1:]):
|
|
0
|
124
|
|
|
125 # parse command line options
|
|
|
126 usage = '%prog [options] manifest [manifest] [...]'
|
|
|
127
|
|
|
128 # description formatter
|
|
|
129 class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
|
|
|
130 def format_description(self, description):
|
|
|
131 if description:
|
|
|
132 return description + '\n'
|
|
|
133 else:
|
|
|
134 return ''
|
|
|
135
|
|
|
136 parser = optparse.OptionParser(usage=usage, description=__doc__, formatter=PlainDescriptionFormatter())
|
|
|
137 parser.add_option('-o', '--output',
|
|
|
138 help="output relative to this location vs. the manifest location")
|
|
|
139 parser.add_option('-d', '--dest',
|
|
|
140 action='append',
|
|
|
141 help="output only these destinations")
|
|
|
142 parser.add_option('-s', '--strict',
|
|
|
143 action='store_true', default=False,
|
|
|
144 help="fail on error")
|
|
|
145 options, args = parser.parse_args(args)
|
|
|
146
|
|
|
147 if not args:
|
|
|
148 parser.print_help()
|
|
|
149 parser.exit()
|
|
|
150
|
|
|
151 items = read_manifests(*args)
|
|
|
152 fetch = Fetch(fetchers, strict=options.strict)
|
|
|
153
|
|
|
154 # download the files
|
|
|
155 fetch.fetch(*items)
|
|
|
156
|
|
|
157 if __name__ == '__main__':
|
|
|
158 main()
|
|
|
159
|
