Mercurial > mozilla > hg > talosnames
annotate talosnames/api.py @ 17:25e91fc7ff01
getting the test config now works
author | Jeff Hammel <jhammel@mozilla.com> |
---|---|
date | Tue, 10 Jul 2012 18:23:16 -0700 |
parents | 1713b3e244a7 |
children | 48b4c926be73 |
rev | line source |
---|---|
12
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
1 import os |
6 | 2 import re |
10 | 3 import require |
13
323a01abd180
putting off this fun for later
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
4 import subprocess |
1 | 5 import sqlite3 |
12
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
6 import sys |
13
323a01abd180
putting off this fun for later
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
7 import talos |
16 | 8 import tempfile |
1 | 9 import urllib2 |
16 | 10 import yaml |
11 | |
12 try: | |
13 call = subprocess.check_call | |
14 except: | |
15 call = subprocess.call | |
1 | 16 |
13
323a01abd180
putting off this fun for later
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
17 talos_dir = os.path.dirname(os.path.abspath(talos.__file__)) |
323a01abd180
putting off this fun for later
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
18 |
12
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
19 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
20 |
0 | 21 class TalosNames(object): |
22 graphserver_sql = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/data.sql' | |
1 | 23 schema = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/schema.sql' |
10 | 24 project_branches = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla/project_branches.py' |
25 buildbot_config = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla-tests/config.py' | |
1 | 26 |
5 | 27 # mapping file from builbot-configs name to tbpl codes: |
28 # http://hg.mozilla.org/users/mstange_themasta.com/tinderboxpushlog/file/tip/js/Config.js | |
6 | 29 tbpl_map = 'http://hg.mozilla.org/users/mstange_themasta.com/tinderboxpushlog/raw-file/tip/js/Data.js' |
5 | 30 |
2 | 31 tables = {'os_list': '(id, name text)', |
3 | 32 'branches': '(id, name text)', |
2 | 33 'machines': '(id, os_id int, is_throttling int, cpu_speed text, name text, is_active int, date_added int)', |
34 'pagesets': '(id, name text)', | |
35 'tests': '(id, name text, pretty_name text, is_chrome int, is_active int, pageset_id int)' | |
36 } | |
1 | 37 |
10 | 38 ### initialization functions |
39 | |
0 | 40 def __init__(self): |
5 | 41 self.setup_database() |
42 self.tbpl_mapping() | |
10 | 43 self.setup_buildbot() |
17
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
44 self._talos_configs = {} |
5 | 45 |
6 | 46 def setup_database(self): |
1 | 47 self.db = sqlite3.connect(':memory:') |
3 | 48 sql_lines = urllib2.urlopen(self.graphserver_sql).readlines() |
49 | |
50 # XXX remove the machines since they require a function, unix_timestamp(), sqlite does not have | |
51 sql_lines = [line for line in sql_lines | |
52 if 'unix_timestamp' not in line] | |
53 sql = '\n'.join(sql_lines) | |
54 | |
1 | 55 cursor = self.db.cursor() |
2 | 56 for table, schema in self.tables.items(): |
57 cursor.execute("""CREATE TABLE %s %s""" % (table, schema)) | |
3 | 58 cursor.executescript(sql) |
2 | 59 self.db.commit() |
60 cursor.close() | |
1 | 61 |
3 | 62 # create data structures |
63 self.names = {} | |
64 self.chrome = set() | |
65 cursor = self.db.cursor() | |
66 cursor.execute("SELECT * FROM tests") | |
67 for _, short_name, graphserver_name, is_chrome, _, _ in cursor.fetchall(): | |
68 self.names[short_name] = graphserver_name | |
69 if is_chrome: | |
70 self.chrome.add(short_name) | |
71 cursor.close() | |
72 | |
5 | 73 def tbpl_mapping(self): |
6 | 74 self.tbpl_regexs = {} |
75 lines = [line.strip() | |
76 for line in urllib2.urlopen(self.tbpl_map).readlines()] | |
77 lines = [line for line in lines | |
78 if line.startswith('/talos.*')] | |
79 regex = re.compile('\/talos\.\*(.*)\/.*\?.*\"([^"].*)\".*') | |
80 for line in lines: | |
81 match = regex.match(line) | |
82 assert match | |
83 _regex, name = match.groups() | |
84 self.tbpl_regexs[name] = _regex | |
5 | 85 |
10 | 86 def setup_buildbot(self): |
87 # project_branches = require.require(self.project_branches) | |
88 module = require.require(self.buildbot_config) | |
89 self.suites = module.SUITES | |
12
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
90 self.buildbot_commands = {} |
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
91 for key, value in self.suites.items(): |
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
92 self.buildbot_commands[key] = value['suites'] |
10 | 93 |
94 ### functions for fetching information | |
95 | |
96 def tbpl_name(self, name): | |
97 """returns the TBPL long name""" | |
98 for tbplname, regex in self.tbpl_regexs.items(): | |
99 regex = re.compile(regex) | |
100 if regex.match(name): | |
101 return tbplname | |
102 | |
12
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
103 def buildbot_command(self, name): |
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
104 """gets the buildbot command for a particular suite""" |
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
105 return self.buildbot_commands.get(name) |
a4aa9f83e3be
adding a thing for commands
Jeff Hammel <jhammel@mozilla.com>
parents:
10
diff
changeset
|
106 |
14 | 107 def talos_config(self, name): |
108 """returns talos configuration for suite `name`""" | |
13
323a01abd180
putting off this fun for later
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
109 command = self.buildbot_command(name) |
323a01abd180
putting off this fun for later
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
110 assert command is not None, "Suite not found: %s" % name |
16 | 111 outfile = tempfile.mktemp(suffix='.yml') |
112 command += ['-o', outfile] # add an output file | |
113 command += ['-e', sys.executable] # add a pretend firefox so PerfConfigurator won't whine | |
114 call(['PerfConfigurator'] + command, stdout=subprocess.PIPE, cwd=talos_dir) | |
115 assert os.path.exists(outfile) | |
116 config = yaml.load(file(outfile)) | |
17
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
117 return config |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
118 |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
119 def test_config(self, name): |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
120 test_config = self.talos_config(name)['tests'] |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
121 retval = {} |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
122 for test in test_config: |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
123 name = test.pop('name') |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
124 retval[name] = test |
25e91fc7ff01
getting the test config now works
Jeff Hammel <jhammel@mozilla.com>
parents:
16
diff
changeset
|
125 return retval |
13
323a01abd180
putting off this fun for later
Jeff Hammel <jhammel@mozilla.com>
parents:
12
diff
changeset
|
126 |
3 | 127 def __call__(self, name=None): |
128 retval = [] | |
129 for short_name, graphserver_name in self.names.items(): | |
130 if (name is None) or (name == short_name or short_name.startswith(name + '_')): | |
131 retval.append((short_name, graphserver_name)) | |
132 retval.sort(key=lambda x: x[0]) | |
133 return retval |