Mercurial > hg > configuration
changeset 62:81996be938bb
parity between module name and package name
| author | Jeff Hammel <jhammel@mozilla.com> |
|---|---|
| date | Tue, 27 Mar 2012 20:17:26 -0700 |
| parents | ef2d1c6211b9 |
| children | e7005b75ef8d |
| files | configuration/__init__.py configuration/config.py configuration/configuration.py |
| diffstat | 3 files changed, 468 insertions(+), 468 deletions(-) [+] |
line wrap: on
line diff
--- a/configuration/__init__.py Tue Mar 27 17:04:31 2012 -0700 +++ b/configuration/__init__.py Tue Mar 27 20:17:26 2012 -0700 @@ -1,2 +1,2 @@ # -from config import * +from configuration import *
--- a/configuration/config.py Tue Mar 27 17:04:31 2012 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,467 +0,0 @@ -#!/usr/bin/env python - -""" -multi-level unified configuration -""" - -import copy -import os -import sys -import optparse - -# imports for contigent configuration providers -try: - import json -except ImportError: - try: - import simplejson as json - except ImportError: - json = None -try: - import yaml -except ImportError: - yaml = None - -__all__ = ['Configuration', 'configuration_providers', 'types', 'MissingValueException', 'ConfigurationProviderException', 'TypeCastException', 'ConfigurationOption'] - -### exceptions - -class MissingValueException(Exception): - """exception raised when a required value is missing""" - -class ConfigurationProviderException(Exception): - """exception raised when a configuration provider is missing, etc""" - -class TypeCastException(Exception): - """exception raised when a configuration item cannot be coerced to a type""" - -### configuration providers for serialization/deserialization - -configuration_providers = [] - -class ConfigurationProvider(object): - """ - abstract base class for configuration providers for - serialization/deserialization - """ - def read(self, filename): - raise NotImplementedError("Abstract base class") - - def write(self, config, filename): - if isinstance(filename, basestring): - f = file(filename, 'w') - newfile = True - else: - f = filename - newfile = False - try: - self._write(f, config) - finally: - # XXX try: finally: works in python >= 2.5 - if newfile: - f.close() - def _write(self, fp, config): - raise NotImplementedError("Abstract base class") - -if json: - class JSON(ConfigurationProvider): - indent = 2 - extensions = ['json'] - def read(self, filename): - return json.loads(file(filename).read()) - def _write(self, fp, config): - fp.write(json.dumps(config, indent=self.indent, sort_keys=True)) - # TODO: could use templates to get order down, etc - configuration_providers.append(JSON()) - -if yaml: - class YAML(ConfigurationProvider): - extensions = ['yml', 'yaml'] - def read(self, filename): - f = file(filename) - config = yaml.load(f) - f.close() - return config - def _write(self, fp, config): - fp.write(yaml.dump(config)) - # TODO: could use templates to get order down, etc - - configuration_providers.append(YAML()) - -# TODO: add a configuration provider for taking command-line arguments -# from a file - -__all__.extend([i.__class__.__name__ for i in configuration_providers]) - -### optparse interface - -class ConfigurationOption(optparse.Option): - """option that keeps track if it is seen""" - # TODO: this should be configurable or something - def take_action(self, action, dest, opt, value, values, parser): - """add the parsed option to the set of things parsed""" - optparse.Option.take_action(self, action, dest, opt, value, values, parser) - if not hasattr(parser, 'parsed'): - parser.parsed = set() - parser.parsed.add(dest) - -### plugins for option types -### TODO: this could use a bit of thought -### They should probably be classes -def base_cli(name, value): - # CLI arguments - args = value.get('flags', ['--%s' % name]) - if not args: - # No CLI interface - return (), {} - - kw = {'dest': name} - help = value.get('help', name) - if 'default' in value: - kw['default'] = value['default'] - # TODO: use default pattern a la - # - http://hg.mozilla.org/build/talos/file/c6013a2f09ce/talos/PerfConfigurator.py#l358 - # - http://k0s.org/mozilla/hg/bzconsole/file/d5e88dadde69/bzconsole/command.py#l12 - - help += ' [DEFAULT: %s]' % value['default'] - kw['help'] = help - kw['action'] = 'store' - return args, kw - -def bool_cli(name, value): - - # preserve the default values - help = value.get('help') - flags = value.get('flags') - - args, kw = base_cli(name, value) - kw['help'] = help # reset - if value.get('default'): - kw['action'] = 'store_false' - if not flags: - args = ['--no-%s' % name] - if not help: - kw['help'] = 'disable %s' % name - else: - kw['action'] = 'store_true' - if not help: - kw['help'] = 'enable %s' % name - return args, kw - -def list_cli(name, value): - args, kw = base_cli(name, value) - - # TODO: could use 'extend' - # - http://hg.mozilla.org/build/mozharness/file/5f44ba08f4be/mozharness/base/config.py#l41 - - # TODO: what about nested types? - kw['action'] = 'append' - return args, kw - -def int_cli(name, value): - args, kw = base_cli(name, value) - kw['type'] = 'int' - return args, kw - -def float_cli(name, value): - args, kw = base_cli(name, value) - kw['type'] = 'float' - return args, kw - -# TODO: 'dict'-type cli interface - -types = {bool: bool_cli, - int: int_cli, - float: float_cli, - list: list_cli, - None: base_cli} # default -__all__ += [i.__name__ for i in types.values()] - -class Configuration(optparse.OptionParser): - """declarative configuration object""" - - options = {} # configuration basis - - def __init__(self, configuration_providers=configuration_providers, types=types, **parser_args): - - # setup configuration - self.config = {} - self.configuration_providers = configuration_providers - self.types = types - - # setup optionparser - if 'description' not in parser_args: - parser_args['description'] = getattr(self, '__doc__', '') - if 'formatter' not in parser_args: - class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): - """description formatter for console script entry point""" - def format_description(self, description): - if description: - return description.strip() + '\n' - else: - return '' - parser_args['formatter'] = PlainDescriptionFormatter() - parser_args.setdefault('option_class', ConfigurationOption) - optparse.OptionParser.__init__(self, **parser_args) - self.parsed = set() - self.optparse_options(self) - - ### methods for iteration - ### TODO: make this a real iterator - - def items(self): - # TODO: allow options to be a list of 2-tuples - return self.options.items() - - ### methods for validating configuration - - def check(self, config): - """ - check validity of configuration to be added - """ - # TODO: should probably deepcopy config - - # ensure options in configuration are in self.options - unknown_options = [i for i in config if i not in self.options] - if unknown_options: - # TODO: more specific error type - raise Exception("Unknown options: %s" % ', '.join(unknown_options)) - - # TODO: ensure options are of the right type (if specified) - for key, value in config.items(): - _type = self.options[key].get('type') - if _type is not None: - try: - config[key] = _type(value) - except BaseException, e: - raise TypeCastException("Could not coerce %s, %s, to type %s: %s" % (key, value, _type.__name__, e)) - - return config - - def validate(self): - """validate resultant configuration""" - for key, value in self.options.items(): - if key not in self.config: - required = value.get('required') - if required: - if isinstance(required, basestring): - required_message = required - else: - required_message = "Parameter %s is required but not present" % key - # TODO: more specific exception - # Also, this should probably raise all missing values vs - # one by one - raise MissingValueException(required_message) - # TODO: configuration should be locked after this is called - - ### methods for adding configuration - - def __call__(self, *args): - """add items to configuration and check it""" - for config in args: - self.add(config) - - # add defaults if not present - for key, value in self.options.items(): - if 'default' in value and key not in self.config: - self.config[key] = value['default'] - - # validate total configuration - self.validate() - # TODO: configuration should be locked after this is called - - def add(self, config, check=True): - """update configuration: not undoable""" - - self.check(config) # check config to be added - self.config.update(config) - # TODO: option to extend; augment lists/dicts - - ### methods for optparse - ### XXX could go in a subclass - - def optparse_options(self, parser): - """add optparse options to a OptionParser instance""" - for key, value in self.items(): - handler = self.types[value.get('type')] - args, kw = handler(key, value) - if not args: - # No CLI interface - continue - parser.add_option(*args, **kw) - - def parser(self, configuration_provider_option=None, dump='--dump', **parser_args): - """ - return OptionParser for this Configuration instance - - configuration_provider_options : option for configuration files - (or '-' for taking from the extensions names) - - dump : option for dumping configuration - - parser_args : arguments to the OptionParser constructor - """ - - if 'description' not in parser_args: - parser_args['description'] = getattr(self, '__doc__', '') - if 'formatter' not in parser_args: - class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): - """description formatter for console script entry point""" - def format_description(self, description): - if description: - return description.strip() + '\n' - else: - return '' - parser_args['formatter'] = PlainDescriptionFormatter() - parser_args.setdefault('option_class', ConfigurationOption) - parser = optparse.OptionParser(**parser_args) - parser.parsed = set() - self.optparse_options(parser) - - # add option(s) for configuration_providers - if configuration_provider_option: - if configuration_provider_option == '-': - raise NotImplementedError(""" -The arguments will be interspersed. Will need to be more clever to get this -to work properly. -""") - - # take option from configuration_provider extensions - for format in self.formats(): - parser.add_option('--%s' % format, - dest='load_%s' % format, - action='append', - help="load configuration from a %s file" % format) - else: - parser.add_option(configuration_provider_option, - dest='load', action='append', - help="load configuration from a file") - - # add an option for dumping - formats = self.formats() - if formats and dump: - parser.add_option(dump, dest='dump', - help="dump configuration to a file; Formats: %s" % formats) - - return parser - - def parse(self, args=sys.argv[1:], parser=None, configuration_provider_option=None): - """ - parse configuration including command line options - - args: command line arguments to parse (default: system arguments) - - parser: a parser instance - - config_provider_option: option for configuration files; if None, - will be taken from remaining args - """ - - # parse arguments - if parser is None: - parser = self.parser(configuration_provider_option=configuration_provider_option) - options, args = parser.parse_args(args) - - # get CLI configuration options - cli_config = dict([(key, value) for key, value in options.__dict__.items() - if key in self.options]) - if hasattr(parser, 'parsed'): - # only use parsed arguments - # (though i'm not sure what to do with parser doesn't have the parsed attribute) - cli_config = dict([(key, value) for key, value in cli_config.items() - if key in parser.parsed]) - - # deserialize configuration - configuration_files = getattr(options, 'load', args) - missing = [i for i in configuration_files - if not os.path.exists(i)] - if missing: - parser.error("Missing files: %s" % ', '.join(missing)) - config = [] - for f in configuration_files: - try: - config.append(self.deserialize(f)) - except BaseException, e: - parser.error(str(e)) - config.append(cli_config) - - missingvalues = None - try: - # generate configuration - self(*config) - except MissingValueException, missingvalues: - pass - - # dump configuration, if specified - dump = getattr(options, 'dump') - if dump: - # TODO: have a way of specifying format other than filename - self.serialize(dump) - - if missingvalues and not dump: - # XXX assuming if you don't have values you were just dumping - raise missingvalues - - # return parsed arguments - return options, args - - ### serialization/deserialization - - def formats(self): - """formats for deserialization""" - retval = [] - for provider in self.configuration_providers: - if provider.extensions and hasattr(provider, 'write'): - retval.append(provider.extensions[0]) - return retval - - def configuration_provider(self, format): - """configuration provider guess for a given filename""" - for provider in self.configuration_providers: - if format in provider.extensions: - return provider - - def filename2format(self, filename): - extension = os.path.splitext(filename)[-1] - return extension.lstrip('.') or None - - def serialize(self, filename, format=None, full=False): - """ - serialize configuration to a file - - filename: path of file to serialize to - - format: format of configuration provider - - full: whether to serialize non-set optional strings [TODO] - """ - # TODO: allow file object vs file name - - if not format: - format = self.filename2format(filename) - if not format: - raise Exception('Please specify a format') - # TODO: more specific exception type - - provider = self.configuration_provider(format) - if not provider: - raise Exception("Provider not found for format: %s" % format) - - config = copy.deepcopy(self.config) - - provider.write(config, filename) - - def deserialize(self, filename, format=None): - """load configuration from a file""" - # TODO: allow file object vs file name - - assert os.path.exists(filename) - - # get the format - if not format: - format = self.filename2format(filename) - - # get the providers in some sensible order - providers = self.configuration_providers[:] - if format: - providers.sort(key=lambda x: int(format in x.extensions), reverse=True) - - # deserialize the data - for provider in providers: - try: - return provider.read(filename) - except: - continue - else: - raise ConfigurationProviderException("Could not load %s" % filename)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configuration/configuration.py Tue Mar 27 20:17:26 2012 -0700 @@ -0,0 +1,467 @@ +#!/usr/bin/env python + +""" +multi-level unified configuration +""" + +import copy +import os +import sys +import optparse + +# imports for contigent configuration providers +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + json = None +try: + import yaml +except ImportError: + yaml = None + +__all__ = ['Configuration', 'configuration_providers', 'types', 'MissingValueException', 'ConfigurationProviderException', 'TypeCastException', 'ConfigurationOption'] + +### exceptions + +class MissingValueException(Exception): + """exception raised when a required value is missing""" + +class ConfigurationProviderException(Exception): + """exception raised when a configuration provider is missing, etc""" + +class TypeCastException(Exception): + """exception raised when a configuration item cannot be coerced to a type""" + +### configuration providers for serialization/deserialization + +configuration_providers = [] + +class ConfigurationProvider(object): + """ + abstract base class for configuration providers for + serialization/deserialization + """ + def read(self, filename): + raise NotImplementedError("Abstract base class") + + def write(self, config, filename): + if isinstance(filename, basestring): + f = file(filename, 'w') + newfile = True + else: + f = filename + newfile = False + try: + self._write(f, config) + finally: + # XXX try: finally: works in python >= 2.5 + if newfile: + f.close() + def _write(self, fp, config): + raise NotImplementedError("Abstract base class") + +if json: + class JSON(ConfigurationProvider): + indent = 2 + extensions = ['json'] + def read(self, filename): + return json.loads(file(filename).read()) + def _write(self, fp, config): + fp.write(json.dumps(config, indent=self.indent, sort_keys=True)) + # TODO: could use templates to get order down, etc + configuration_providers.append(JSON()) + +if yaml: + class YAML(ConfigurationProvider): + extensions = ['yml', 'yaml'] + def read(self, filename): + f = file(filename) + config = yaml.load(f) + f.close() + return config + def _write(self, fp, config): + fp.write(yaml.dump(config)) + # TODO: could use templates to get order down, etc + + configuration_providers.append(YAML()) + +# TODO: add a configuration provider for taking command-line arguments +# from a file + +__all__.extend([i.__class__.__name__ for i in configuration_providers]) + +### optparse interface + +class ConfigurationOption(optparse.Option): + """option that keeps track if it is seen""" + # TODO: this should be configurable or something + def take_action(self, action, dest, opt, value, values, parser): + """add the parsed option to the set of things parsed""" + optparse.Option.take_action(self, action, dest, opt, value, values, parser) + if not hasattr(parser, 'parsed'): + parser.parsed = set() + parser.parsed.add(dest) + +### plugins for option types +### TODO: this could use a bit of thought +### They should probably be classes +def base_cli(name, value): + # CLI arguments + args = value.get('flags', ['--%s' % name]) + if not args: + # No CLI interface + return (), {} + + kw = {'dest': name} + help = value.get('help', name) + if 'default' in value: + kw['default'] = value['default'] + # TODO: use default pattern a la + # - http://hg.mozilla.org/build/talos/file/c6013a2f09ce/talos/PerfConfigurator.py#l358 + # - http://k0s.org/mozilla/hg/bzconsole/file/d5e88dadde69/bzconsole/command.py#l12 + + help += ' [DEFAULT: %s]' % value['default'] + kw['help'] = help + kw['action'] = 'store' + return args, kw + +def bool_cli(name, value): + + # preserve the default values + help = value.get('help') + flags = value.get('flags') + + args, kw = base_cli(name, value) + kw['help'] = help # reset + if value.get('default'): + kw['action'] = 'store_false' + if not flags: + args = ['--no-%s' % name] + if not help: + kw['help'] = 'disable %s' % name + else: + kw['action'] = 'store_true' + if not help: + kw['help'] = 'enable %s' % name + return args, kw + +def list_cli(name, value): + args, kw = base_cli(name, value) + + # TODO: could use 'extend' + # - http://hg.mozilla.org/build/mozharness/file/5f44ba08f4be/mozharness/base/config.py#l41 + + # TODO: what about nested types? + kw['action'] = 'append' + return args, kw + +def int_cli(name, value): + args, kw = base_cli(name, value) + kw['type'] = 'int' + return args, kw + +def float_cli(name, value): + args, kw = base_cli(name, value) + kw['type'] = 'float' + return args, kw + +# TODO: 'dict'-type cli interface + +types = {bool: bool_cli, + int: int_cli, + float: float_cli, + list: list_cli, + None: base_cli} # default +__all__ += [i.__name__ for i in types.values()] + +class Configuration(optparse.OptionParser): + """declarative configuration object""" + + options = {} # configuration basis + + def __init__(self, configuration_providers=configuration_providers, types=types, **parser_args): + + # setup configuration + self.config = {} + self.configuration_providers = configuration_providers + self.types = types + + # setup optionparser + if 'description' not in parser_args: + parser_args['description'] = getattr(self, '__doc__', '') + if 'formatter' not in parser_args: + class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): + """description formatter for console script entry point""" + def format_description(self, description): + if description: + return description.strip() + '\n' + else: + return '' + parser_args['formatter'] = PlainDescriptionFormatter() + parser_args.setdefault('option_class', ConfigurationOption) + optparse.OptionParser.__init__(self, **parser_args) + self.parsed = set() + self.optparse_options(self) + + ### methods for iteration + ### TODO: make this a real iterator + + def items(self): + # TODO: allow options to be a list of 2-tuples + return self.options.items() + + ### methods for validating configuration + + def check(self, config): + """ + check validity of configuration to be added + """ + # TODO: should probably deepcopy config + + # ensure options in configuration are in self.options + unknown_options = [i for i in config if i not in self.options] + if unknown_options: + # TODO: more specific error type + raise Exception("Unknown options: %s" % ', '.join(unknown_options)) + + # TODO: ensure options are of the right type (if specified) + for key, value in config.items(): + _type = self.options[key].get('type') + if _type is not None: + try: + config[key] = _type(value) + except BaseException, e: + raise TypeCastException("Could not coerce %s, %s, to type %s: %s" % (key, value, _type.__name__, e)) + + return config + + def validate(self): + """validate resultant configuration""" + for key, value in self.options.items(): + if key not in self.config: + required = value.get('required') + if required: + if isinstance(required, basestring): + required_message = required + else: + required_message = "Parameter %s is required but not present" % key + # TODO: more specific exception + # Also, this should probably raise all missing values vs + # one by one + raise MissingValueException(required_message) + # TODO: configuration should be locked after this is called + + ### methods for adding configuration + + def __call__(self, *args): + """add items to configuration and check it""" + for config in args: + self.add(config) + + # add defaults if not present + for key, value in self.options.items(): + if 'default' in value and key not in self.config: + self.config[key] = value['default'] + + # validate total configuration + self.validate() + # TODO: configuration should be locked after this is called + + def add(self, config, check=True): + """update configuration: not undoable""" + + self.check(config) # check config to be added + self.config.update(config) + # TODO: option to extend; augment lists/dicts + + ### methods for optparse + ### XXX could go in a subclass + + def optparse_options(self, parser): + """add optparse options to a OptionParser instance""" + for key, value in self.items(): + handler = self.types[value.get('type')] + args, kw = handler(key, value) + if not args: + # No CLI interface + continue + parser.add_option(*args, **kw) + + def parser(self, configuration_provider_option=None, dump='--dump', **parser_args): + """ + return OptionParser for this Configuration instance + - configuration_provider_options : option for configuration files + (or '-' for taking from the extensions names) + - dump : option for dumping configuration + - parser_args : arguments to the OptionParser constructor + """ + + if 'description' not in parser_args: + parser_args['description'] = getattr(self, '__doc__', '') + if 'formatter' not in parser_args: + class PlainDescriptionFormatter(optparse.IndentedHelpFormatter): + """description formatter for console script entry point""" + def format_description(self, description): + if description: + return description.strip() + '\n' + else: + return '' + parser_args['formatter'] = PlainDescriptionFormatter() + parser_args.setdefault('option_class', ConfigurationOption) + parser = optparse.OptionParser(**parser_args) + parser.parsed = set() + self.optparse_options(parser) + + # add option(s) for configuration_providers + if configuration_provider_option: + if configuration_provider_option == '-': + raise NotImplementedError(""" +The arguments will be interspersed. Will need to be more clever to get this +to work properly. +""") + + # take option from configuration_provider extensions + for format in self.formats(): + parser.add_option('--%s' % format, + dest='load_%s' % format, + action='append', + help="load configuration from a %s file" % format) + else: + parser.add_option(configuration_provider_option, + dest='load', action='append', + help="load configuration from a file") + + # add an option for dumping + formats = self.formats() + if formats and dump: + parser.add_option(dump, dest='dump', + help="dump configuration to a file; Formats: %s" % formats) + + return parser + + def parse(self, args=sys.argv[1:], parser=None, configuration_provider_option=None): + """ + parse configuration including command line options + - args: command line arguments to parse (default: system arguments) + - parser: a parser instance + - config_provider_option: option for configuration files; if None, + will be taken from remaining args + """ + + # parse arguments + if parser is None: + parser = self.parser(configuration_provider_option=configuration_provider_option) + options, args = parser.parse_args(args) + + # get CLI configuration options + cli_config = dict([(key, value) for key, value in options.__dict__.items() + if key in self.options]) + if hasattr(parser, 'parsed'): + # only use parsed arguments + # (though i'm not sure what to do with parser doesn't have the parsed attribute) + cli_config = dict([(key, value) for key, value in cli_config.items() + if key in parser.parsed]) + + # deserialize configuration + configuration_files = getattr(options, 'load', args) + missing = [i for i in configuration_files + if not os.path.exists(i)] + if missing: + parser.error("Missing files: %s" % ', '.join(missing)) + config = [] + for f in configuration_files: + try: + config.append(self.deserialize(f)) + except BaseException, e: + parser.error(str(e)) + config.append(cli_config) + + missingvalues = None + try: + # generate configuration + self(*config) + except MissingValueException, missingvalues: + pass + + # dump configuration, if specified + dump = getattr(options, 'dump') + if dump: + # TODO: have a way of specifying format other than filename + self.serialize(dump) + + if missingvalues and not dump: + # XXX assuming if you don't have values you were just dumping + raise missingvalues + + # return parsed arguments + return options, args + + ### serialization/deserialization + + def formats(self): + """formats for deserialization""" + retval = [] + for provider in self.configuration_providers: + if provider.extensions and hasattr(provider, 'write'): + retval.append(provider.extensions[0]) + return retval + + def configuration_provider(self, format): + """configuration provider guess for a given filename""" + for provider in self.configuration_providers: + if format in provider.extensions: + return provider + + def filename2format(self, filename): + extension = os.path.splitext(filename)[-1] + return extension.lstrip('.') or None + + def serialize(self, filename, format=None, full=False): + """ + serialize configuration to a file + - filename: path of file to serialize to + - format: format of configuration provider + - full: whether to serialize non-set optional strings [TODO] + """ + # TODO: allow file object vs file name + + if not format: + format = self.filename2format(filename) + if not format: + raise Exception('Please specify a format') + # TODO: more specific exception type + + provider = self.configuration_provider(format) + if not provider: + raise Exception("Provider not found for format: %s" % format) + + config = copy.deepcopy(self.config) + + provider.write(config, filename) + + def deserialize(self, filename, format=None): + """load configuration from a file""" + # TODO: allow file object vs file name + + assert os.path.exists(filename) + + # get the format + if not format: + format = self.filename2format(filename) + + # get the providers in some sensible order + providers = self.configuration_providers[:] + if format: + providers.sort(key=lambda x: int(format in x.extensions), reverse=True) + + # deserialize the data + for provider in providers: + try: + return provider.read(filename) + except: + continue + else: + raise ConfigurationProviderException("Could not load %s" % filename)