# HG changeset patch # User Jeff Hammel # Date 1341962578 25200 # Node ID 1029ddf7b8062694315eb126994c6130ad9082fa # Parent ab2bd5dda72caa59383f8d2c94e03cc4b6508d0d get the buildbot config diff -r ab2bd5dda72c -r 1029ddf7b806 talosnames/api.py --- a/talosnames/api.py Tue Jul 10 14:54:03 2012 -0700 +++ b/talosnames/api.py Tue Jul 10 16:22:58 2012 -0700 @@ -1,10 +1,13 @@ import re +import require import sqlite3 import urllib2 class TalosNames(object): graphserver_sql = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/data.sql' schema = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/schema.sql' + project_branches = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla/project_branches.py' + buildbot_config = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla-tests/config.py' # mapping file from builbot-configs name to tbpl codes: # http://hg.mozilla.org/users/mstange_themasta.com/tinderboxpushlog/file/tip/js/Config.js @@ -17,9 +20,12 @@ 'tests': '(id, name text, pretty_name text, is_chrome int, is_active int, pageset_id int)' } + ### initialization functions + def __init__(self): self.setup_database() self.tbpl_mapping() + self.setup_buildbot() def setup_database(self): self.db = sqlite3.connect(':memory:') @@ -61,6 +67,20 @@ _regex, name = match.groups() self.tbpl_regexs[name] = _regex + def setup_buildbot(self): +# project_branches = require.require(self.project_branches) + module = require.require(self.buildbot_config) + self.suites = module.SUITES + + ### functions for fetching information + + def tbpl_name(self, name): + """returns the TBPL long name""" + for tbplname, regex in self.tbpl_regexs.items(): + regex = re.compile(regex) + if regex.match(name): + return tbplname + def __call__(self, name=None): retval = [] for short_name, graphserver_name in self.names.items(): diff -r ab2bd5dda72c -r 1029ddf7b806 talosnames/localconfig.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/talosnames/localconfig.py Tue Jul 10 16:22:58 2012 -0700 @@ -0,0 +1,100 @@ +SLAVES = { + 'fedora': dict([("talos-r3-fed-%03i" % x, {}) for x in range(3,10) + range(11,18) + range(19,77)]), + 'fedora64' : dict([("talos-r3-fed64-%03i" % x, {}) for x in range (3,10) + range(11,35) + range(36,72)]), + 'xp': dict([("talos-r3-xp-%03i" % x, {}) for x in range(4,10) + range(11,76) \ + if x not in [45, 59]]), # bug 661377, bug 753357 + 'win7': dict([("talos-r3-w7-%03i" % x, {}) for x in range(4,10) + range(11,80)]), + 'w764': dict([("t-r3-w764-%03i" % x, {}) for x in range(1,6)]), + 'leopard': dict([("talos-r3-leopard-%03i" % x, {}) for x in range(3,10) + range(11,67) \ + if x not in [7]]), # bug 655437 + 'snowleopard': dict([("talos-r4-snow-%03i" % x, {}) for x in range(4,10) + range(11,81) + [82,84]]), + 'lion': dict([("talos-r4-lion-%03i" % x, {}) for x in range(4,10) + range(11,83) + [84]]), + 'tegra_android': dict([('tegra-%03i' % x, {'http_port': '30%03i' % x, 'ssl_port': '31%03i' % x}) \ + for x in range(31,289) \ + if x not in range(122,129) + [30,31,33,34,43,44,49,65,69,77,131,137,143,147,\ + 153,156,161,175,176,180,184,185,186,193,197,198,202,203,204,205,222,224,\ + 226,239,241,268,275,289]]), # decommissioned tegras +} + +SLAVES['leopard-o'] = SLAVES['leopard'] +SLAVES['tegra_android-xul'] = SLAVES['tegra_android'] +SLAVES['tegra_android-o'] = SLAVES['tegra_android'] + +TRY_SLAVES = {} + +GRAPH_CONFIG = ['--resultsServer', 'graphs.mozilla.org', + '--resultsLink', '/server/collect.cgi'] + +GLOBAL_VARS = { + 'disable_tinderbox_mail': True, + 'build_tools_repo_path': 'build/tools', + 'stage_server': 'stage.mozilla.org', + 'stage_username': 'ffxbld', + 'stage_ssh_key': 'ffxbld_dsa', +} + +# Local branch overrides +BRANCHES = { + 'mozilla-central': { + 'tinderbox_tree': 'Firefox', + 'mobile_tinderbox_tree': 'Firefox', + }, + 'mozilla-release': { + 'tinderbox_tree': 'Mozilla-Release', + 'mobile_tinderbox_tree': 'Mozilla-Release', + }, + 'mozilla-esr10': { + 'tinderbox_tree': 'Mozilla-Esr10', + 'mobile_tinderbox_tree': 'Mozilla-Esr10', + }, + 'mozilla-beta': { + 'tinderbox_tree': 'Mozilla-Beta', + 'mobile_tinderbox_tree': 'Mozilla-Beta', + }, + 'mozilla-aurora': { + 'tinderbox_tree': 'Mozilla-Aurora', + 'mobile_tinderbox_tree': 'Mozilla-Aurora', + }, + 'places': { + 'tinderbox_tree': 'Places', + 'mobile_tinderbox_tree': 'Places', + }, + 'electrolysis': { + 'tinderbox_tree': 'Electrolysis', + 'mobile_tinderbox_tree': 'Electrolysis', + }, + 'addontester': { + 'tinderbox_tree': 'AddonTester', + 'mobile_tinderbox_tree': 'AddonTester', + }, + 'addonbaselinetester': { + 'tinderbox_tree': 'AddonTester', + 'mobile_tinderbox_tree': 'AddonTester', + }, + 'try': { + 'tinderbox_tree': 'Try', + 'mobile_tinderbox_tree': 'Try', + 'enable_mail_notifier': True, + 'notify_real_author': True, + 'enable_merging': False, + 'slave_key': 'try_slaves', + 'package_url': 'http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds', + 'package_dir': '%(who)s-%(got_revision)s', + 'stage_username': 'trybld', + 'stage_ssh_key': 'trybld_dsa', + }, + 'jaegermonkey': { + 'tinderbox_tree': 'Jaegermonkey', + 'mobile_tinderbox_tree': 'Jaegermonkey', + }, +} + +PLATFORM_VARS = { +} + +PROJECTS = { + 'jetpack': { + 'scripts_repo': 'http://hg.mozilla.org/build/tools', + 'tinderbox_tree': 'Jetpack', + }, +} diff -r ab2bd5dda72c -r 1029ddf7b806 talosnames/main.py --- a/talosnames/main.py Tue Jul 10 14:54:03 2012 -0700 +++ b/talosnames/main.py Tue Jul 10 16:22:58 2012 -0700 @@ -15,6 +15,7 @@ self.api = api.TalosNames() def graphserver(self, *names): + """give graphserver names from test names""" if not names: names = [None] for arg in names: @@ -22,6 +23,10 @@ print '%s : %s' % (name, graphserver_name) print + def tbpl(self, name): + """give TBPL name given the buildbot config name""" + return self.api.tbpl_name(name) + def main(args=sys.argv[1:]): parser = CommandParser(TalosNamesCLI) parser.invoke(sys.argv[1:]) diff -r ab2bd5dda72c -r 1029ddf7b806 talosnames/project_branches.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/talosnames/project_branches.py Tue Jul 10 16:22:58 2012 -0700 @@ -0,0 +1,291 @@ +# Additional branches that start as identical (individual variables can be overriden here) +PROJECT_BRANCHES = { + ### PLEASE ADD NEW BRANCHES ALPHABETICALLY (twigs at the bottom, also alphabetically) + 'accessibility': { + 'mozconfig_dir': 'accessibility', + 'enable_nightly': True, + 'enabled_products': ['firefox'], + # only want a11y which is run within the "chrome" suite + # turn other suites off + 'talos_suites': { + 'dirty': 0, + 'tp4': 0, + 'tp': 0, + 'chrome_twinopen': 0, + 'chrome_mac': 0, + 'chrome': 0, + 'nochrome': 0, + 'dromaeo': 0, + 'svg': 0, + 'paint': 0, + }, + 'add_test_suites': [ + ('macosx64', 'snowleopard', 'opt', 'mochitest-other', 'mochitest-a11y'), + ('macosx64', 'snowleopard', 'debug', 'mochitest-other', 'mochitest-a11y'), + ] + }, + 'build-system': { + 'pgo_strategy': 'per-checkin', + 'platforms': { + 'win32': { + 'pgo_platform': 'win64', + }, + }, + }, + 'devtools':{ + 'enable_nightly': True, + 'enabled_products': ['firefox'], + 'platforms': { + 'macosx-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + 'macosx': { + 'slave_platforms': [], + }, + 'macosx64': { + 'slave_platforms': ['snowleopard', 'lion'], + }, + 'android': { + 'enable_opt_unittests': False, + 'enable_debug_unittests': False, + 'tegra_android': {}, + }, + }, + }, + # DISABLED because of builder limit problems - bug 721854 + #'electrolysis': { + # 'mozconfig_dir': 'electrolysis', + # 'enable_talos': True, + #}, + 'fx-team': { + 'repo_path': 'integration/fx-team', + 'mozconfig_dir': 'mozilla-central', + 'enable_nightly': True, + 'pgo_strategy': 'periodic', + }, + # Turning off graphics - bug 649507 + #'graphics':{ + # 'enable_unittests': False, + # 'enable_talos': False, + #}, + 'ionmonkey': { + 'mozconfig_dir': 'mozilla-central', + 'enable_nightly': True, + 'create_snippet': True, + 'create_partial': True, + 'pgo_strategy': 'periodic', + 'talos_suites': { + 'v8': 1, + }, + }, + 'jaegermonkey': { + 'mozconfig_dir': 'jaegermonkey', + 'enable_nightly': True, + 'create_snippet': True, + 'create_partial': True, + }, + 'mozilla-inbound': { + 'repo_path': 'integration/mozilla-inbound', + 'mozconfig_dir': 'mozilla-central', + 'enable_nightly': True, + 'enable_weekly_bundle': True, + 'pgo_strategy': 'periodic', + 'periodic_pgo_interval': 3, + 'platforms': { + 'linux64': { + 'build_space': 7, + 'nightly_signing_servers': 'nightly-signing', + }, + 'linux': { + 'build_space': 7, + 'nightly_signing_servers': 'nightly-signing', + }, + 'linuxqt': { + 'build_space': 7, + }, + 'macosx64-debug': { + 'enable_leaktests': True, + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'macosx-debug': { + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'macosx64': { + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'win32': { + 'nightly_signing_servers': 'nightly-signing', + }, + }, + 'talos_suites': { + 'v8': 1, + } + }, + # DISABLED because of builder limit problems - bug 721854 +# 'places': { +# 'platforms': { +# 'linux64': { +# 'build_space': 6, +# }, +# 'linux': { +# 'build_space': 6, +# }, +# 'linuxqt': { +# 'build_space': 6, +# }, +# }, +# }, + 'profiling': { + 'enable_talos': False, + 'platforms': { + 'macosx-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'macosx64-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'macosx64': { + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'linux': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'linux64': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'linux-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + 'linux64-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + 'win32-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + 'win32': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'win64': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'android-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + }, + }, + 'services-central': { + 'repo_path': 'services/services-central', + 'enable_weekly_bundle': True, + 'pgo_strategy': 'periodic', + }, + 'ux': { + 'branch_name': 'UX', + 'mobile_branch_name': 'UX', + 'build_branch': 'UX', + 'tinderbox_tree': 'UX', + 'mobile_tinderbox_tree': 'UX', + 'packaged_unittest_tinderbox_tree': 'UX', + 'enabled_products': ['firefox'], + 'mozconfig_dir' : 'ux', + 'enable_nightly': True, + 'create_snippet': True, + 'create_partial': True, + 'platforms': { + 'macosx-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'macosx64-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'macosx64': { + 'nightly_signing_servers': 'mac-nightly-signing', + }, + 'linux': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'linux64': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'linux-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + 'linux64-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + 'win32-debug': { + 'dont_build': True, + 'enable_debug_unittests': False, + }, + }, + }, + ##### TWIGS aka RENTABLE BRANCHES + # customizations while booked for bug 687570 - WebRTC project + 'alder': {}, + 'ash': {}, + 'birch': {}, + 'cedar': {}, + # customizations for windows update service changes (bug 481815) + 'elm': { + 'enable_nightly': True, + 'create_snippet': True, + 'create_partial': True, + 'lock_platforms': True, + 'platforms': { + 'win32': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'win64': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'win32-debug': {}, + 'win32-metro': {}, + }, + 'enable_talos': False, + }, + 'holly': {}, + 'larch': {}, + 'maple': {}, + # customizations for integration work for bugs 481815 and 307181 + 'oak': { + 'enable_nightly': True, + 'create_snippet': True, + 'create_partial': True, + 'enable_talos': False, + 'platforms': { + 'linux': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'linux64': { + 'nightly_signing_servers': 'nightly-signing', + }, + 'win32': { + 'nightly_signing_servers': 'nightly-signing', + }, + }, + }, + 'pine': {}, +} + +# All is the default +ACTIVE_PROJECT_BRANCHES = PROJECT_BRANCHES.keys() + +# Load up project branches' local values +for branch in PROJECT_BRANCHES.keys(): + PROJECT_BRANCHES[branch]['tinderbox_tree'] = PROJECT_BRANCHES[branch].get('tinderbox_tree', branch.title()) + PROJECT_BRANCHES[branch]['mobile_tinderbox_tree'] = PROJECT_BRANCHES[branch].get('mobile_tinderbox_tree', branch.title()) + PROJECT_BRANCHES[branch]['packaged_unittest_tinderbox_tree'] = PROJECT_BRANCHES[branch].get('packaged_unittest_tinderbox_tree', branch.title()) diff -r ab2bd5dda72c -r 1029ddf7b806 talosnames/require.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/talosnames/require.py Tue Jul 10 16:22:58 2012 -0700 @@ -0,0 +1,19 @@ +def require(url): + """ + import a module from the web + url should be like scheme://host.name/path/to/module.py + """ + import imp + import os + import tempfile + import urllib2 + contents = urllib2.urlopen(url).read() + filename = url.rsplit('/', 1)[-1] + module = filename.rsplit('.', 1)[-1] + dest = tempfile.mktemp(suffix='.py', prefix=module) + f = file(dest, 'w') + f.write(contents) + f.close() + return imp.load_source(module, dest) + +# TODO: make an equivalent method for a tarball