Mercurial > mozilla > hg > talosnames
view talosnames/api.py @ 42:cc0edf61ab21
require all the things!
author | Jeff Hammel <jhammel@mozilla.com> |
---|---|
date | Sun, 12 Aug 2012 23:10:16 -0700 |
parents | 4dfa9c298e3d |
children | 601ebbbdf63e |
line wrap: on
line source
import os import re import require import subprocess import sqlite3 import sys import talos import talos.run_tests import tempfile import urllib2 import yaml try: call = subprocess.check_call except: call = subprocess.call talos_dir = os.path.dirname(os.path.abspath(talos.__file__)) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) class TalosNames(object): # files for graphserver graphserver_sql = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/data.sql' schema = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/schema.sql' # files for buildbot project_branches = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla/project_branches.py' buildbot_config = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla-tests/config.py' localconfig = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla-tests/production_config.py' # mapping file from builbot-configs name to tbpl codes: # http://hg.mozilla.org/users/mstange_themasta.com/tinderboxpushlog/file/tip/js/Config.js tbpl_map = 'http://hg.mozilla.org/users/mstange_themasta.com/tinderboxpushlog/raw-file/tip/js/Data.js' tables = {'os_list': '(id, name text)', 'branches': '(id, name text)', 'machines': '(id, os_id int, is_throttling int, cpu_speed text, name text, is_active int, date_added int)', 'pagesets': '(id, name text)', 'tests': '(id, name text, pretty_name text, is_chrome int, is_active int, pageset_id int)' } ### initialization functions def __init__(self): self.setup_database() self.tbpl_mapping() self.setup_buildbot() # cache self._talos_configs = {} self._tbpl_names = {} def setup_database(self): self.db = sqlite3.connect(':memory:') sql_lines = urllib2.urlopen(self.graphserver_sql).readlines() # XXX remove the machines since they require a function, unix_timestamp(), sqlite does not have sql_lines = [line for line in sql_lines if 'unix_timestamp' not in line] sql = '\n'.join(sql_lines) cursor = self.db.cursor() for table, schema in self.tables.items(): cursor.execute("""CREATE TABLE %s %s""" % (table, schema)) cursor.executescript(sql) self.db.commit() cursor.close() # create data structures self.names = {} self.chrome = set() cursor = self.db.cursor() cursor.execute("SELECT * FROM tests") for _, short_name, graphserver_name, is_chrome, _, _ in cursor.fetchall(): self.names[short_name] = graphserver_name if is_chrome: self.chrome.add(short_name) cursor.close() def tbpl_mapping(self): self.tbpl_regexs = {} lines = [line.strip() for line in urllib2.urlopen(self.tbpl_map).readlines()] lines = [line for line in lines if line.startswith('/talos.*')] regex = re.compile('\/talos\.\*(.*)\/.*\?.*\"([^"].*)\".*') for line in lines: match = regex.match(line) assert match _regex, name = match.groups() self.tbpl_regexs[name] = _regex def setup_buildbot(self): localconfig, project_branches, module = require.require(self.localconfig, self.project_branches, self.buildbot_config, **{self.localconfig: "localconfig.py"}) self.suites = module.SUITES self.buildbot_commands = {} self.buildbot_enabled = {} for key, value in self.suites.items(): self.buildbot_commands[key] = value['suites'] self.buildbot_enabled[key] = value['enable_by_default'] ### functions for fetching information def tbpl_name(self, name): """returns the TBPL long name""" if name in self._tbpl_names: return self._tbpl_names[name] for tbplname, regex in self.tbpl_regexs.items(): regex = re.compile(regex) if regex.match(name): self._tbpl_names[name] = tbplname return tbplname def buildbot_command(self, name): """gets the buildbot command for a particular suite""" return self.buildbot_commands.get(name)[:] def talos_config(self, name): """returns talos configuration for suite `name`""" if name in self._talos_configs: return self._talos_configs[name] command = self.buildbot_command(name) assert command is not None, "Suite not found: %s" % name outfile = tempfile.mktemp(suffix='.yml') command += ['-o', outfile] # add an output file command += ['-e', sys.executable] # add a pretend firefox so PerfConfigurator won't whine print "\n%s" % subprocess.list2cmdline(command) call(['PerfConfigurator'] + command, stdout=subprocess.PIPE, cwd=talos_dir) assert os.path.exists(outfile) config = yaml.load(file(outfile)) self._talos_configs[name] = config return config def test_config(self, name): config = self.talos_config(name) test_config = config['tests'] talos.run_tests.useBaseTestDefaults(config.get('basetest', {}), test_config) retval = {} for test in test_config: name = test.pop('name') retval[name] = test return retval def __call__(self, name=None): """returns the graphserver name prefixed with `name`""" retval = [] for short_name, graphserver_name in self.names.items(): if (name is None) or (name == short_name or short_name.startswith(name + '_')): retval.append((short_name, graphserver_name)) retval.sort(key=lambda x: x[0]) return retval def graphserver_name(self, name): for short_name, graphserver_name in self.names.items(): if name == short_name: return (short_name, graphserver_name)