Mercurial > mozilla > hg > dogfood
annotate dogdish/dispatcher.py @ 15:fc5b842de4e9
be a little less aggressive when caching
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Wed, 17 Oct 2012 14:59:49 -0700 |
| parents | cffb6f681b59 |
| children | 31898fb3b7d9 |
| rev | line source |
|---|---|
| 1 | 1 #!/usr/bin/env python |
| 2 | |
| 0 | 3 """ |
| 1 | 4 dogdish |
|
7
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
5 https://bugzilla.mozilla.org/show_bug.cgi?id=800118 |
| 0 | 6 """ |
| 7 | |
|
12
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
8 import fnmatch |
| 14 | 9 import hashlib |
| 8 | 10 import os |
| 4 | 11 import sys |
| 0 | 12 from urlparse import urlparse |
| 4 | 13 from webob import Request |
| 0 | 14 from webob import Response, exc |
| 11 | 15 from ConfigParser import RawConfigParser as ConfigParser |
| 0 | 16 |
| 8 | 17 here = os.path.dirname(os.path.abspath(__file__)) |
| 18 | |
|
15
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
19 ### models |
| 11 | 20 |
| 21 class Application(object): | |
| 22 """class for storing application.ini data""" | |
| 23 | |
| 24 def __init__(self, filename): | |
| 25 """ | |
| 26 - filename : path to an application.ini file | |
| 27 """ | |
| 14 | 28 config = ConfigParser() |
| 29 config.read(filename) | |
| 30 self.build_id = config.get('App', 'BuildID') | |
| 31 self.version = config.get('App', 'Version') | |
| 11 | 32 |
|
15
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
33 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
34 class Update(object): |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
35 """class representing a .mar update file""" |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
36 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
37 prefix = 'b2g_update_' |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
38 suffix = '.mar' |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
39 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
40 @classmethod |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
41 def updates(cls, directory): |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
42 """returns the updates in a directory""" |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
43 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
44 contents = [i for i in os.listdir(directory) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
45 if i.startswith(cls.prefix) and i.endswith(cls.suffix)] |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
46 contents = set(contents) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
47 return contents |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
48 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
49 def __init__(self, directory, filename): |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
50 self.directory = directory |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
51 self.filename = filename |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
52 self.path = os.path.join(directory, filename) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
53 self.stamp = filename[len(self.prefix):-len(self.suffix)] |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
54 self.size = os.path.getsize(self.path) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
55 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
56 # cached properties |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
57 self._application = None |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
58 self._hash = None |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
59 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
60 def application(self): |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
61 """ |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
62 returns the path to the application.ini |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
63 associated with this update |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
64 """ |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
65 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
66 if not self._application: |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
67 application_ini = 'application_%s.ini' % self.stamp |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
68 application_ini = os.path.join(self.directory, application_ini) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
69 assert os.path.exists(application_ini) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
70 self._application = Application(application_ini) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
71 return self._application |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
72 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
73 def hash(self): |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
74 if not self._hash: |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
75 # compute the hash |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
76 with file(self.path) as f: |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
77 self._hash = hashlib.sha512(f.read()).hexdigest() |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
78 return self._hash |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
79 |
| 11 | 80 ### request handlers |
| 81 | |
| 0 | 82 class Handler(object): |
| 9 | 83 """abstract handler object for a request""" |
| 1 | 84 |
| 14 | 85 def __init__(self, app, request): |
| 86 self.app = app | |
| 0 | 87 self.request = request |
| 88 self.application_path = urlparse(request.application_url)[2] | |
| 89 | |
| 90 def link(self, path=(), permanant=False): | |
| 91 if isinstance(path, basestring): | |
| 92 path = [ path ] | |
| 93 path = [ i.strip('/') for i in path ] | |
| 94 if permanant: | |
| 95 application_url = [ self.request.application_url ] | |
| 96 else: | |
| 97 application_url = [ self.application_path ] | |
| 98 path = application_url + path | |
| 99 return '/'.join(path) | |
| 100 | |
| 101 def redirect(self, location): | |
| 102 raise exc.HTTPSeeOther(location=location) | |
| 103 | |
| 104 class Get(Handler): | |
| 9 | 105 """handle GET requests""" |
| 0 | 106 |
| 9 | 107 # template for response body |
| 0 | 108 body = """<?xml version="1.0"?> |
| 109 <updates> | |
| 110 <update type="minor" appVersion="19.0a1" version="19.0a1" extensionVersion="19.0a1" buildID="20121010114416" licenseURL="http://www.mozilla.com/test/sample-eula.html" detailsURL="http://www.mozilla.com/test/sample-details.html"> | |
| 14 | 111 <patch type="complete" URL="http://update.boot2gecko.org/nightly/%(update)s%(query)s" hashFunction="SHA512" hashValue="%(hash)s" size="%(size)s"/> |
| 0 | 112 </update> |
| 113 </updates>""" | |
| 114 | |
| 9 | 115 ### methods for request handler |
| 116 | |
| 0 | 117 @classmethod |
| 118 def match(cls, request): | |
| 119 return request.method == 'GET' | |
| 120 | |
| 121 def __call__(self): | |
| 14 | 122 |
|
7
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
123 body = self.body |
|
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
124 query = {} |
|
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
125 dogfood_id = self.request.GET.get('dogfood_id') |
|
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
126 if dogfood_id: |
|
12
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
127 query['dogfooding_prerelease_id'] = dogfood_id |
|
15
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
128 current_update = self.app.current_update |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
129 application = current_update.application() |
| 14 | 130 query['build_id'] = application.build_id |
| 131 query['version'] = application.version | |
|
7
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
132 |
|
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
133 # build query string |
|
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
134 if query: |
| 14 | 135 query = '?' + '&'.join(['%s=%s' % (key, value) for key, value in query.items()]) |
|
7
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
136 else: |
|
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
137 query = '' |
|
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
138 |
| 14 | 139 |
|
12
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
140 # template variables |
|
15
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
141 variables = dict(update=current_update.filename, |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
142 size=current_update.size, |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
143 hash=current_update.hash(), |
| 14 | 144 query=query) |
|
12
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
145 |
| 1 | 146 return Response(content_type='text/xml', |
|
12
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
147 body=body % variables) |
| 0 | 148 |
| 149 class Dispatcher(object): | |
| 9 | 150 """web application""" |
| 151 | |
| 0 | 152 |
| 153 ### class level variables | |
| 8 | 154 defaults = {'directory': here} |
| 0 | 155 |
| 156 def __init__(self, **kw): | |
| 11 | 157 |
| 158 # set defaults | |
| 0 | 159 for key in self.defaults: |
| 160 setattr(self, key, kw.get(key, self.defaults[key])) | |
| 161 self.handlers = [ Get ] | |
| 14 | 162 |
| 163 # cache | |
| 11 | 164 self.updates = {} |
| 165 self.current_update = None | |
| 0 | 166 |
| 11 | 167 # scan directory |
| 168 self.scan() | |
| 14 | 169 assert self.current_update # ensure there is at least one update |
| 11 | 170 |
| 0 | 171 def __call__(self, environ, start_response): |
| 11 | 172 """WSGI application""" |
| 173 | |
| 0 | 174 request = Request(environ) |
| 14 | 175 new = self.scan() |
| 0 | 176 for h in self.handlers: |
| 177 if h.match(request): | |
| 14 | 178 handler = h(self, request) |
| 179 res = handler() | |
| 0 | 180 break |
| 181 else: | |
| 14 | 182 res = exc.HTTPNotFound() |
| 183 | |
| 0 | 184 return res(environ, start_response) |
| 185 | |
| 11 | 186 def scan(self): |
| 14 | 187 """ |
| 188 scan the directory for updates | |
| 189 returns True if new updates are found, False otherwise | |
| 190 """ | |
|
15
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
191 |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
192 # check for new updates |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
193 contents = Update.updates(self.directory) |
|
12
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
194 new = contents.difference(self.updates.keys()) |
|
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
195 if not new: |
|
234c2427e52b
beginnings of a scan function
Jeff Hammel <jhammel@mozilla.com>
parents:
11
diff
changeset
|
196 # directory contents unchanged from cached values |
| 14 | 197 return False |
| 11 | 198 |
|
13
71f9f68986b5
directory scanning + caching
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
199 for update in new: |
|
15
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
200 self.updates[update] = Update(self.directory, update) |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
201 if self.current_update: |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
202 if self.updates[update].stamp > self.current_update.stamp: |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
203 self.current_update = self.updates[update] |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
204 else: |
|
fc5b842de4e9
be a little less aggressive when caching
Jeff Hammel <jhammel@mozilla.com>
parents:
14
diff
changeset
|
205 self.current_update = self.updates[update] |
|
13
71f9f68986b5
directory scanning + caching
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
206 |
| 14 | 207 # TODO: could remove old files from the cache if not found in contents |
| 208 | |
| 209 return True | |
| 210 | |
| 0 | 211 def main(args=sys.argv[1:]): |
| 212 """CLI entry point""" | |
| 213 | |
| 11 | 214 # imports for CLI |
| 0 | 215 import optparse |
| 216 from wsgiref import simple_server | |
| 217 | |
| 11 | 218 # parse CLI options |
| 0 | 219 parser = optparse.OptionParser() |
| 220 parser.add_option('-p', '--port', dest='port', | |
| 221 default=8080, type='int', | |
| 222 help="port to serve on") | |
| 11 | 223 parser.add_option('-d', '--directory', dest='directory', |
| 224 default=os.getcwd(), | |
| 225 help="directory of update files") | |
| 0 | 226 options, args = parser.parse_args() |
| 227 | |
| 11 | 228 # create the app |
| 229 app = Dispatcher(directory=options.directory) | |
| 0 | 230 |
| 11 | 231 # serve the app |
|
7
65d0a1153864
a stub that sorta does something
Jeff Hammel <jhammel@mozilla.com>
parents:
6
diff
changeset
|
232 print "http://localhost:%s/" % options.port |
| 0 | 233 server = simple_server.make_server(host='0.0.0.0', port=options.port, app=app) |
| 234 server.serve_forever() | |
| 235 | |
| 236 if __name__ == '__main__': | |
| 237 main() |