Mercurial > mozilla > hg > talosnames

changeset 10:1029ddf7b806

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
get the buildbot config
author Jeff Hammel <jhammel@mozilla.com>
date Tue, 10 Jul 2012 16:22:58 -0700
parents ab2bd5dda72c
children b7b59561be5e
files talosnames/api.py talosnames/localconfig.py talosnames/main.py talosnames/project_branches.py talosnames/require.py
diffstat 5 files changed, 435 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/talosnames/api.py	Tue Jul 10 14:54:03 2012 -0700
+++ b/talosnames/api.py	Tue Jul 10 16:22:58 2012 -0700
@@ -1,10 +1,13 @@
 import re
+import require
 import sqlite3
 import urllib2
 
 class TalosNames(object):
     graphserver_sql = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/data.sql'
     schema = 'http://hg.mozilla.org/graphs/raw-file/tip/sql/schema.sql'
+    project_branches = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla/project_branches.py'
+    buildbot_config = 'http://hg.mozilla.org/build/buildbot-configs/raw-file/tip/mozilla-tests/config.py'
 
     # mapping file from builbot-configs name to tbpl codes:
     # http://hg.mozilla.org/users/mstange_themasta.com/tinderboxpushlog/file/tip/js/Config.js
@@ -17,9 +20,12 @@
               'tests': '(id, name text, pretty_name text, is_chrome int, is_active int, pageset_id int)'
               }
 
+    ### initialization functions
+
     def __init__(self):
         self.setup_database()
         self.tbpl_mapping()
+        self.setup_buildbot()
 
     def setup_database(self):
         self.db = sqlite3.connect(':memory:')
@@ -61,6 +67,20 @@
             _regex, name = match.groups()
             self.tbpl_regexs[name] = _regex
 
+    def setup_buildbot(self):
+#        project_branches = require.require(self.project_branches)
+        module = require.require(self.buildbot_config)
+        self.suites = module.SUITES
+
+    ### functions for fetching information
+
+    def tbpl_name(self, name):
+        """returns the TBPL long name"""
+        for tbplname, regex in self.tbpl_regexs.items():
+            regex = re.compile(regex)
+            if regex.match(name):
+                return tbplname
+
     def __call__(self, name=None):
         retval = []
         for short_name, graphserver_name in self.names.items():
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/talosnames/localconfig.py	Tue Jul 10 16:22:58 2012 -0700
@@ -0,0 +1,100 @@
+SLAVES = {
+    'fedora': dict([("talos-r3-fed-%03i" % x, {}) for x in range(3,10) + range(11,18) + range(19,77)]),
+    'fedora64' : dict([("talos-r3-fed64-%03i" % x, {}) for x in range (3,10) + range(11,35) + range(36,72)]),
+    'xp': dict([("talos-r3-xp-%03i" % x, {}) for x in range(4,10) + range(11,76) \
+          if x not in [45, 59]]), # bug 661377, bug 753357
+    'win7': dict([("talos-r3-w7-%03i" % x, {}) for x in range(4,10) + range(11,80)]),
+    'w764': dict([("t-r3-w764-%03i" % x, {}) for x in range(1,6)]),
+    'leopard': dict([("talos-r3-leopard-%03i" % x, {}) for x in range(3,10) + range(11,67) \
+          if x not in [7]]), # bug 655437
+    'snowleopard': dict([("talos-r4-snow-%03i" % x, {}) for x in range(4,10) + range(11,81) + [82,84]]),
+    'lion': dict([("talos-r4-lion-%03i" % x, {}) for x in range(4,10) + range(11,83) + [84]]),
+    'tegra_android': dict([('tegra-%03i' % x, {'http_port': '30%03i' % x, 'ssl_port': '31%03i' % x}) \
+        for x in range(31,289) \
+        if x not in range(122,129) + [30,31,33,34,43,44,49,65,69,77,131,137,143,147,\
+            153,156,161,175,176,180,184,185,186,193,197,198,202,203,204,205,222,224,\
+            226,239,241,268,275,289]]), # decommissioned tegras
+}
+
+SLAVES['leopard-o'] = SLAVES['leopard']
+SLAVES['tegra_android-xul'] = SLAVES['tegra_android']
+SLAVES['tegra_android-o'] = SLAVES['tegra_android']
+
+TRY_SLAVES = {}
+
+GRAPH_CONFIG = ['--resultsServer', 'graphs.mozilla.org',
+    '--resultsLink', '/server/collect.cgi']
+
+GLOBAL_VARS = {
+    'disable_tinderbox_mail': True,
+    'build_tools_repo_path': 'build/tools',
+    'stage_server': 'stage.mozilla.org',
+    'stage_username': 'ffxbld',
+    'stage_ssh_key': 'ffxbld_dsa',
+}
+
+# Local branch overrides
+BRANCHES = {
+    'mozilla-central': {
+        'tinderbox_tree': 'Firefox',
+        'mobile_tinderbox_tree': 'Firefox',
+    },
+    'mozilla-release': {
+        'tinderbox_tree': 'Mozilla-Release',
+        'mobile_tinderbox_tree': 'Mozilla-Release',
+    },
+    'mozilla-esr10': {
+        'tinderbox_tree': 'Mozilla-Esr10',
+        'mobile_tinderbox_tree': 'Mozilla-Esr10',
+    },
+    'mozilla-beta': {
+        'tinderbox_tree': 'Mozilla-Beta',
+        'mobile_tinderbox_tree': 'Mozilla-Beta',
+    },
+    'mozilla-aurora': {
+        'tinderbox_tree': 'Mozilla-Aurora',
+        'mobile_tinderbox_tree': 'Mozilla-Aurora',
+    },
+    'places': {
+        'tinderbox_tree': 'Places',
+        'mobile_tinderbox_tree': 'Places',
+    },
+    'electrolysis': {
+        'tinderbox_tree': 'Electrolysis',
+        'mobile_tinderbox_tree': 'Electrolysis',
+    },
+    'addontester': {
+        'tinderbox_tree': 'AddonTester',
+        'mobile_tinderbox_tree': 'AddonTester',
+    },
+    'addonbaselinetester': {
+        'tinderbox_tree': 'AddonTester',
+        'mobile_tinderbox_tree': 'AddonTester',
+    },
+    'try': {
+        'tinderbox_tree': 'Try',
+        'mobile_tinderbox_tree': 'Try',
+        'enable_mail_notifier': True,
+        'notify_real_author': True,
+        'enable_merging': False,
+        'slave_key': 'try_slaves',
+        'package_url': 'http://ftp.mozilla.org/pub/mozilla.org/firefox/try-builds',
+        'package_dir': '%(who)s-%(got_revision)s',
+        'stage_username': 'trybld',
+        'stage_ssh_key': 'trybld_dsa',
+    },
+    'jaegermonkey': {
+        'tinderbox_tree': 'Jaegermonkey',
+        'mobile_tinderbox_tree': 'Jaegermonkey',
+    },
+}
+
+PLATFORM_VARS = {
+}
+
+PROJECTS = {
+    'jetpack': {
+        'scripts_repo': 'http://hg.mozilla.org/build/tools',
+        'tinderbox_tree': 'Jetpack',
+    },
+}
--- a/talosnames/main.py	Tue Jul 10 14:54:03 2012 -0700
+++ b/talosnames/main.py	Tue Jul 10 16:22:58 2012 -0700
@@ -15,6 +15,7 @@
         self.api = api.TalosNames()
 
     def graphserver(self, *names):
+        """give graphserver names from test names"""
         if not names:
             names = [None]
         for arg in names:
@@ -22,6 +23,10 @@
                 print '%s : %s' % (name, graphserver_name)
             print
 
+    def tbpl(self, name):
+        """give TBPL name given the buildbot config name"""
+        return self.api.tbpl_name(name)
+
 def main(args=sys.argv[1:]):
     parser = CommandParser(TalosNamesCLI)
     parser.invoke(sys.argv[1:])
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/talosnames/project_branches.py	Tue Jul 10 16:22:58 2012 -0700
@@ -0,0 +1,291 @@
+# Additional branches that start as identical (individual variables can be overriden here)
+PROJECT_BRANCHES = {
+    ### PLEASE ADD NEW BRANCHES ALPHABETICALLY (twigs at the bottom, also alphabetically)
+    'accessibility': {
+        'mozconfig_dir': 'accessibility',
+        'enable_nightly': True,
+        'enabled_products': ['firefox'],
+        # only want a11y which is run within the "chrome" suite
+        # turn other suites off
+        'talos_suites': {
+            'dirty': 0,
+            'tp4': 0,
+            'tp': 0,
+            'chrome_twinopen': 0,
+            'chrome_mac': 0,
+            'chrome': 0,
+            'nochrome': 0,
+            'dromaeo': 0,
+            'svg': 0,
+            'paint': 0,
+        },
+        'add_test_suites': [
+            ('macosx64', 'snowleopard', 'opt', 'mochitest-other', 'mochitest-a11y'),
+            ('macosx64', 'snowleopard', 'debug', 'mochitest-other', 'mochitest-a11y'),
+        ]
+    },
+    'build-system': {
+        'pgo_strategy': 'per-checkin',
+        'platforms': {
+            'win32': {
+                'pgo_platform': 'win64',
+            },
+        },
+    },
+    'devtools':{
+        'enable_nightly': True,
+        'enabled_products': ['firefox'],
+        'platforms': {
+            'macosx-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+            'macosx': {
+                'slave_platforms': [],
+            },
+            'macosx64': {
+                'slave_platforms': ['snowleopard', 'lion'],
+            },
+            'android': {
+                'enable_opt_unittests': False,
+                'enable_debug_unittests': False,
+                'tegra_android': {},
+            },
+        },
+    },
+    # DISABLED because of builder limit problems - bug 721854
+    #'electrolysis': {
+    #    'mozconfig_dir': 'electrolysis',
+    #    'enable_talos': True,
+    #},
+    'fx-team': {
+        'repo_path': 'integration/fx-team',
+        'mozconfig_dir': 'mozilla-central',
+        'enable_nightly': True,
+        'pgo_strategy': 'periodic',
+    },
+    # Turning off graphics - bug 649507
+    #'graphics':{
+    #    'enable_unittests': False,
+    #    'enable_talos': False,
+    #},
+    'ionmonkey': {
+        'mozconfig_dir': 'mozilla-central',
+        'enable_nightly': True,
+        'create_snippet': True,
+        'create_partial': True,
+        'pgo_strategy': 'periodic',
+        'talos_suites': {
+            'v8': 1,
+        },
+    },
+    'jaegermonkey': {
+        'mozconfig_dir': 'jaegermonkey',
+        'enable_nightly': True,
+        'create_snippet': True,
+        'create_partial': True,
+    },
+    'mozilla-inbound': {
+        'repo_path': 'integration/mozilla-inbound',
+        'mozconfig_dir': 'mozilla-central',
+        'enable_nightly': True,
+        'enable_weekly_bundle': True,
+        'pgo_strategy': 'periodic',
+        'periodic_pgo_interval': 3,
+        'platforms': {
+            'linux64': {
+                'build_space': 7,
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'linux': {
+                'build_space': 7,
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'linuxqt': {
+                'build_space': 7,
+            },
+            'macosx64-debug': {
+                'enable_leaktests': True,
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'macosx-debug': {
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'macosx64': {
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'win32': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+        },
+        'talos_suites': {
+            'v8': 1,
+        }
+    },
+    # DISABLED because of builder limit problems - bug 721854
+#    'places': {
+#        'platforms': {
+#            'linux64': {
+#                'build_space': 6,
+#            },
+#            'linux': {
+#                'build_space': 6,
+#            },
+#            'linuxqt': {
+#                'build_space': 6,
+#            },
+#        },
+#    },
+    'profiling': {
+        'enable_talos': False,
+        'platforms': {
+            'macosx-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'macosx64-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'macosx64': {
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'linux': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'linux64': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'linux-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+            'linux64-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+            'win32-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+            'win32': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'win64': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'android-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+        },
+    },
+    'services-central': {
+        'repo_path': 'services/services-central',
+        'enable_weekly_bundle': True,
+        'pgo_strategy': 'periodic',
+    },
+    'ux': {
+        'branch_name': 'UX',
+        'mobile_branch_name': 'UX',
+        'build_branch': 'UX',
+        'tinderbox_tree': 'UX',
+        'mobile_tinderbox_tree': 'UX',
+        'packaged_unittest_tinderbox_tree': 'UX',
+        'enabled_products': ['firefox'],
+        'mozconfig_dir' : 'ux',
+        'enable_nightly': True,
+        'create_snippet': True,
+        'create_partial': True,
+        'platforms': {
+            'macosx-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'macosx64-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'macosx64': {
+                'nightly_signing_servers': 'mac-nightly-signing',
+            },
+            'linux': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'linux64': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'linux-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+            'linux64-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+            'win32-debug': {
+                'dont_build': True,
+                'enable_debug_unittests': False,
+            },
+        },
+    },
+    #####  TWIGS aka RENTABLE BRANCHES
+    # customizations while booked for bug 687570 - WebRTC project
+    'alder': {},
+    'ash': {},
+    'birch': {},
+    'cedar': {},
+    # customizations for windows update service changes (bug 481815)
+    'elm': {
+        'enable_nightly': True,
+        'create_snippet': True,
+        'create_partial': True,
+        'lock_platforms': True,
+        'platforms': {
+            'win32': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'win64': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'win32-debug': {},
+            'win32-metro': {},
+        },
+        'enable_talos': False,
+    },
+    'holly': {},
+    'larch': {},
+    'maple': {},
+    # customizations for integration work for bugs 481815 and 307181
+    'oak': {
+        'enable_nightly': True,
+        'create_snippet': True,
+        'create_partial': True,
+        'enable_talos': False,
+        'platforms': {
+            'linux': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'linux64': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+            'win32': {
+                'nightly_signing_servers': 'nightly-signing',
+            },
+        },
+    },
+    'pine': {},
+}
+
+# All is the default
+ACTIVE_PROJECT_BRANCHES = PROJECT_BRANCHES.keys()
+
+# Load up project branches' local values
+for branch in PROJECT_BRANCHES.keys():
+    PROJECT_BRANCHES[branch]['tinderbox_tree'] = PROJECT_BRANCHES[branch].get('tinderbox_tree', branch.title())
+    PROJECT_BRANCHES[branch]['mobile_tinderbox_tree'] = PROJECT_BRANCHES[branch].get('mobile_tinderbox_tree', branch.title())
+    PROJECT_BRANCHES[branch]['packaged_unittest_tinderbox_tree'] = PROJECT_BRANCHES[branch].get('packaged_unittest_tinderbox_tree', branch.title())
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/talosnames/require.py	Tue Jul 10 16:22:58 2012 -0700
@@ -0,0 +1,19 @@
+def require(url):
+    """
+    import a module from the web
+    url should be like scheme://host.name/path/to/module.py
+    """
+    import imp
+    import os
+    import tempfile
+    import urllib2
+    contents = urllib2.urlopen(url).read()
+    filename = url.rsplit('/', 1)[-1]
+    module = filename.rsplit('.', 1)[-1]
+    dest = tempfile.mktemp(suffix='.py', prefix=module)
+    f = file(dest, 'w')
+    f.write(contents)
+    f.close()
+    return imp.load_source(module, dest)
+
+# TODO: make an equivalent method for a tarball