140
|
1 #!/usr/bin/env python
|
|
2
|
|
3 """
|
|
4 unified configuration with serialization/deserialization
|
|
5 """
|
|
6
|
|
7 # imports
|
|
8 import argparse
|
|
9 import copy
|
|
10 import os
|
|
11 import sys
|
|
12
|
|
13 # imports for configuration providers
|
|
14 try:
|
|
15 import json
|
|
16 except ImportError:
|
|
17 try:
|
|
18 import simplejson as json
|
|
19 except ImportError:
|
|
20 json = None
|
|
21 try:
|
|
22 import yaml
|
|
23 except ImportError:
|
|
24 yaml = None
|
|
25
|
|
26 # module contents
|
|
27 __all__ = ['Configuration',
|
|
28 'configuration_providers',
|
|
29 'types',
|
|
30 'UnknownOptionException',
|
|
31 'MissingValueException',
|
|
32 'ConfigurationProviderException',
|
|
33 'TypeCastException',
|
|
34 ]
|
|
35
|
|
36
|
|
37
|
|
38 ### exceptions
|
|
39
|
|
40 class UnknownOptionException(Exception):
|
|
41 """exception raised when a non-configuration value is present in the configuration"""
|
|
42
|
|
43 class MissingValueException(Exception):
|
|
44 """exception raised when a required value is missing"""
|
|
45
|
|
46 class ConfigurationProviderException(Exception):
|
|
47 """exception raised when a configuration provider is missing, etc"""
|
|
48
|
|
49 class TypeCastException(Exception):
|
|
50 """exception raised when a configuration item cannot be coerced to a type"""
|
|
51
|
|
52
|
|
53 ### configuration providers for serialization/deserialization
|
|
54
|
|
55 configuration_providers = []
|
|
56
|
|
57 class ConfigurationProvider(object):
|
|
58 """
|
|
59 abstract base class for configuration providers for
|
|
60 serialization/deserialization
|
|
61 """
|
|
62 def read(self, filename):
|
|
63 raise NotImplementedError("Abstract base class")
|
|
64
|
|
65 def write(self, config, filename):
|
|
66 if isinstance(filename, basestring):
|
|
67 f = file(filename, 'w')
|
|
68 newfile = True
|
|
69 else:
|
|
70 f = filename
|
|
71 newfile = False
|
|
72 try:
|
|
73 self._write(f, config)
|
|
74 finally:
|
|
75 if newfile:
|
|
76 f.close()
|
|
77 def _write(self, fp, config):
|
|
78 raise NotImplementedError("Abstract base class")
|
|
79
|
|
80 if json:
|
|
81 class JSON(ConfigurationProvider):
|
|
82 indent = 2
|
|
83 extensions = ['json']
|
|
84 def read(self, filename):
|
|
85 return json.loads(file(filename).read())
|
|
86 def _write(self, fp, config):
|
|
87 fp.write(json.dumps(config, indent=self.indent, sort_keys=True))
|
|
88 configuration_providers.append(JSON())
|
|
89
|
|
90 if yaml:
|
|
91 class YAML(ConfigurationProvider):
|
|
92 extensions = ['yml', 'yaml']
|
|
93 dump_args = {'default_flow_style': False}
|
|
94 def read(self, filename):
|
|
95 f = file(filename)
|
|
96 config = yaml.load(f)
|
|
97 f.close()
|
|
98 return config
|
|
99 def _write(self, fp, config):
|
|
100 fp.write(yaml.dump(config, **self.dump_args))
|
|
101 # TODO: could use templates to get order down, etc
|
|
102
|
|
103 configuration_providers.append(YAML())
|
|
104
|
|
105 # TODO: add configuration providers
|
|
106 # - for taking command-line arguments from a file
|
|
107 # - for .ini files
|
|
108
|
|
109 __all__.extend([i.__class__.__name__ for i in configuration_providers])
|
|
110
|
|
111
|
|
112 ### optparse interface
|
|
113
|
|
114 #class ConfigurationOption(optparse.Option):
|
|
115 # """option that keeps track if it is seen"""
|
|
116 # # TODO: this should be configurable or something
|
|
117 # def take_action(self, action, dest, opt, value, values, parser):
|
|
118 #
|
|
119 # # switch on types
|
|
120 # formatter = getattr(parser, 'cli_formatter')
|
|
121 # if formatter:
|
|
122 # formatter = formatter(dest)
|
|
123 # if formatter:
|
|
124 # value = formatter(value)
|
|
125
|
|
126 # call the optparse front-end
|
|
127 # optparse.Option.take_action(self, action, dest, opt, value, values, parser)
|
|
128
|
|
129 # # add the parsed option to the set of things parsed
|
|
130 # if not hasattr(parser, 'parsed'):
|
|
131 # parser.parsed = dict()
|
|
132 # parser.parsed[dest] = value
|
|
133
|
|
134
|
|
135 ### plugins for option types
|
|
136
|
|
137 class BaseCLI(object):
|
|
138 """base_cli for all option types"""
|
|
139
|
|
140 def __call__(self, name, value):
|
|
141 """return args, kwargs needed to instantiate an optparse option"""
|
|
142
|
|
143 args = value.get('flags', ['--%s' % name])
|
|
144 if not args:
|
|
145 # No CLI interface
|
|
146 return (), {}
|
|
147
|
|
148 kw = {'dest': name}
|
|
149 help = value.get('help', name)
|
|
150 if 'default' in value:
|
|
151 kw['default'] = value['default']
|
|
152 help += ' [DEFAULT: %(default)s]'
|
|
153 kw['help'] = help
|
|
154 kw['action'] = 'store'
|
|
155 return args, kw
|
|
156
|
|
157 def take_action(self, value):
|
|
158 return value
|
|
159
|
|
160
|
|
161 class BoolCLI(BaseCLI):
|
|
162
|
|
163 def __call__(self, name, value):
|
|
164
|
|
165 # preserve the default values
|
|
166 help = value.get('help')
|
|
167 flags = value.get('flags')
|
|
168
|
|
169 args, kw = BaseCLI.__call__(self, name, value)
|
|
170 kw['help'] = help # reset
|
|
171 if value.get('default'):
|
|
172 kw['action'] = 'store_false'
|
|
173 if not flags:
|
|
174 args = ['--no-%s' % name]
|
|
175 if not help:
|
|
176 kw['help'] = 'disable {}'.format(name)
|
|
177 else:
|
|
178 kw['action'] = 'store_true'
|
|
179 if not help:
|
|
180 kw['help'] = 'enable {}'.format(name)
|
|
181 return args, kw
|
|
182
|
|
183 class IntCLI(BaseCLI):
|
|
184
|
|
185 def __call__(self, name, value):
|
|
186 pass # TODO
|
|
187
|
|
188 class ListCLI(BaseCLI):
|
|
189
|
|
190 def __call__(self, name, value):
|
|
191 args, kw = BaseCLI.__call__(self, name, value)
|
|
192 kw['nargs'] = '+'
|
|
193 return args, kw
|
|
194
|
|
195
|
|
196 class DictCLI(ListCLI):
|
|
197
|
|
198 delimeter = '='
|
|
199
|
|
200 def __call__(self, name, value):
|
|
201
|
|
202 # optparse can't handle dict types OOTB
|
|
203 default = value.get('default')
|
|
204 if isinstance(default, dict):
|
|
205 value = copy.deepcopy(value)
|
|
206 value['default'] = default.items()
|
|
207
|
|
208 return ListCLI.__call__(self, name, value)
|
|
209
|
|
210 def take_action(self, value):
|
|
211 if self.delimeter not in value:
|
|
212 raise AssertionError("Each value must be delimited by '%s': %s" % (self.delimeter, value))
|
|
213 return value.split(self.delimeter, 1)
|
|
214
|
|
215 # types of CLI arguments
|
|
216 types = {bool: BoolCLI(),
|
|
217 list: ListCLI(),
|
|
218 dict: DictCLI(),
|
|
219 str: BaseCLI(),
|
|
220 None: BaseCLI()} # default
|
|
221
|
|
222 __all__ += [i.__class__.__name__ for i in types.values()]
|
|
223
|
|
224
|
|
225 class Configuration(argparse.ArgumentParser):
|
|
226 """declarative configuration object"""
|
|
227
|
|
228 options = {} # configuration basis definition
|
|
229 extend = set() # which dicts/lists should be extended
|
|
230 load_option = 'load' # where to put the load option
|
|
231 load_help = "load configuration from a file"
|
|
232
|
|
233 @classmethod
|
|
234 def parse(cls, args, *_args, **_kwargs):
|
|
235 """get the resultant config dictionary in a single call"""
|
|
236 conf = cls(*_args, **_kwargs)
|
|
237 conf.parse_args(*_args, **_kwargs)
|
|
238 return conf.config
|
|
239
|
|
240 def __init__(self,
|
|
241 configuration_providers=configuration_providers,
|
|
242 types=types,
|
|
243 load=None,
|
|
244 dump='--dump',
|
|
245 **parser_args):
|
|
246
|
|
247 # sanity check
|
|
248 if isinstance(self.options, dict):
|
|
249 self.option_dict = self.options
|
|
250 elif isinstance(self.options, (list, tuple)):
|
|
251 self.option_dict = dict(self.options)
|
|
252 else:
|
|
253 raise NotImplementedError("Configuration: `options` should be castable to a dict")
|
|
254
|
|
255 # setup configuration
|
|
256 self.config = {}
|
|
257 self.configuration_providers = configuration_providers
|
|
258 self.types = types
|
|
259 self.added = set() # set of items added to the configuration
|
|
260
|
|
261
|
|
262 if 'description' not in parser_args:
|
|
263 parser_args['description'] = getattr(self, '__doc__', '')
|
|
264 # if 'formatter' not in parser_args:
|
|
265 # class PlainDescriptionFormatter(optparse.IndentedHelpFormatter):
|
|
266 # """description formatter for console script entry point"""
|
|
267 # def format_description(self, description):
|
|
268 # if description:
|
|
269 # return description.strip() + '\n'
|
|
270 # else:
|
|
271 # return ''
|
|
272 # parser_args['formatter'] = PlainDescriptionFormatter()
|
|
273 # parser_args.setdefault('option_class', ConfigurationOption)
|
|
274
|
|
275 # setup commandline parser
|
|
276 argparse.ArgumentParser.__init__(self, **parser_args)
|
|
277 self.parsed = dict()
|
|
278 self.add_arguments(self)
|
|
279
|
|
280 # add option(s) for configuration_providers
|
|
281 if load:
|
|
282 self.add_argument(load,
|
|
283 dest=self.load_option,
|
|
284 nargs='+',
|
|
285 help=self.load_help)
|
|
286 elif load is None:
|
|
287 self.add_argument(self.load_option, nargs='*',
|
|
288 help=self.load_help)
|
|
289
|
|
290 # add an option for dumping
|
|
291 formats = self.formats()
|
|
292 if formats and dump:
|
|
293 if isinstance(dump, basestring):
|
|
294 dump = [dump]
|
|
295 dump = list(dump)
|
|
296 self.add_argument(*dump,
|
|
297 **dict(dest='dump',
|
|
298 help="Output configuration file; Formats: {}".format(formats)))
|
|
299
|
|
300
|
|
301 ### iteration
|
|
302
|
|
303 def items(self):
|
|
304 """items in options"""
|
|
305 # TODO: make the class a real iterator
|
|
306
|
|
307 # allow options to be a list of 2-tuples
|
|
308 if isinstance(self.options, dict):
|
|
309 return self.options.items()
|
|
310 return self.options
|
|
311
|
|
312 ### methods for validating configuration
|
|
313
|
|
314 def check(self, config):
|
|
315 """
|
|
316 check validity of configuration to be added
|
|
317 """
|
|
318
|
|
319 # ensure options in configuration are in self.options
|
|
320 unknown_options = [i for i in config if i not in self.option_dict]
|
|
321 if unknown_options:
|
|
322 raise UnknownOptionException("Unknown options: {}".format(', '.join(unknown_options)))
|
|
323
|
|
324 # ensure options are of the right type (if specified)
|
|
325 for key, value in config.items():
|
|
326 _type = self.option_dict[key].get('type')
|
|
327 if _type is None and 'default' in self.option_dict[key]:
|
|
328 _type = type(self.option_dict[key]['default'])
|
|
329 if _type is not None:
|
|
330 tocast = True
|
|
331 try:
|
|
332 if isinstance(value, _type):
|
|
333 tocast = False
|
|
334 except TypeError:
|
|
335 # type is a type-casting function, not a proper type
|
|
336 pass
|
|
337 if tocast:
|
|
338 try:
|
|
339 config[key] = _type(value)
|
|
340 except BaseException, e:
|
|
341 raise TypeCastException("Could not coerce '%s'=%s, to type %s: %s" % (key, value, _type.__name__, e))
|
|
342
|
|
343 return config
|
|
344
|
|
345 def validate(self):
|
|
346 """validate resultant configuration"""
|
|
347
|
|
348 for key, value in self.items():
|
|
349 if key not in self.config:
|
|
350 required = value.get('required')
|
|
351 if required:
|
|
352 if isinstance(required, basestring):
|
|
353 required_message = required
|
|
354 else:
|
|
355 required_message = "Parameter {} is required but not present".format(key)
|
|
356 # TODO: this should probably raise all missing values vs
|
|
357 # one by one
|
|
358 raise MissingValueException(required_message)
|
|
359
|
|
360
|
|
361 ### methods for adding configuration
|
|
362
|
|
363 def default_config(self):
|
|
364 """configuration defaults"""
|
|
365 defaults = {}
|
|
366 for key, value in self.items():
|
|
367 if 'default' in value:
|
|
368 defaults[key] = value['default']
|
|
369 return copy.deepcopy(defaults)
|
|
370
|
|
371 def get(self, key, default=None):
|
|
372 return self.config.get(key, default)
|
|
373
|
|
374 def __getitem__(self, key):
|
|
375 return self.config[key]
|
|
376
|
|
377 def __call__(self, *args):
|
|
378 """add items to configuration and check it"""
|
|
379
|
|
380 # start with defaults
|
|
381 self.config = self.default_config()
|
|
382
|
|
383 # add the configuration
|
|
384 for config in args:
|
|
385 self.add(config)
|
|
386
|
|
387 # validate total configuration
|
|
388 self.validate()
|
|
389
|
|
390 # return the configuration
|
|
391 return self.config
|
|
392
|
|
393 def add(self, config, check=True):
|
|
394 """update configuration: not undoable"""
|
|
395
|
|
396 # check config to be added
|
|
397 self.check(config)
|
|
398
|
|
399 # add the configuration
|
|
400 for key, value in config.items():
|
|
401 value = copy.deepcopy(value)
|
|
402 if key in self.extend and key in self.config:
|
|
403 type1 = type(self.config[key])
|
|
404 type2 = type(value)
|
|
405 assert type1 == type2 # XXX hack
|
|
406 if type1 == dict:
|
|
407 self.config[key].update(value)
|
|
408 elif type1 == list:
|
|
409 self.config[key].extend(value)
|
|
410 else:
|
|
411 raise NotImplementedError
|
|
412 else:
|
|
413 self.config[key] = value
|
|
414 self.added.add(key)
|
|
415
|
|
416
|
|
417 ### methods for argparse
|
|
418 ### XXX could go in a subclass
|
|
419
|
|
420 def help_formatter(self, option):
|
|
421 if option in self.option_dict:
|
|
422 handler = self.types[self.option_type(option)]
|
|
423 return getattr(handler, 'take_action', lambda x: x)
|
|
424
|
|
425 def option_type(self, name):
|
|
426 """get the type of an option named `name`"""
|
|
427
|
|
428 value = self.option_dict[name]
|
|
429 if 'type' in value:
|
|
430 return value['type']
|
|
431 if 'default' in value:
|
|
432 default = value['default']
|
|
433 if default is None:
|
|
434 return None # not <type 'NoneType'>
|
|
435 return type(value['default'])
|
|
436
|
|
437 def add_arguments(self, parser):
|
|
438 """add argparse arguments to an ArgumentParser instance"""
|
|
439
|
|
440 for key, value in self.items():
|
|
441 try:
|
|
442 handler = self.types[self.option_type(key)]
|
|
443 except KeyError:
|
|
444 # if an option can't be coerced to a type
|
|
445 # we should just not add a CLI handler for it
|
|
446 continue
|
|
447 args, kw = handler(key, value)
|
|
448 if not args:
|
|
449 # No CLI interface
|
|
450 continue
|
|
451 parser.add_argument(*args, **kw)
|
|
452
|
|
453
|
|
454 ### functions for loading configuration
|
|
455
|
|
456 def configuration_files(self, options):
|
|
457 """configuration files to read"""
|
|
458
|
|
459 configuration_files = getattr(options, self.load_option, [])
|
|
460 if isinstance(configuration_files, basestring):
|
|
461 configuration_files = [configuration_files]
|
|
462 return configuration_files
|
|
463
|
|
464 def load_configuration_file(self, filename):
|
|
465 """load a configuration file"""
|
|
466 return self.deserialize(filename)
|
|
467
|
|
468 def read_configuration_files(self, options):
|
|
469 """deserialize configuration"""
|
|
470
|
|
471 configuration_files = self.configuration_files(options)
|
|
472
|
|
473 # ensure all files are present
|
|
474 missing = [i for i in configuration_files
|
|
475 if not os.path.exists(i)]
|
|
476 if missing:
|
|
477 self.error("Missing files: {}".format(', '.join(missing)))
|
|
478
|
|
479 # load configuration files
|
|
480 config = []
|
|
481 for f in configuration_files:
|
|
482 try:
|
|
483 loaded_config = self.load_configuration_file(f)
|
|
484 if loaded_config:
|
|
485 config.append(loaded_config)
|
|
486 except BaseException, e:
|
|
487 parser.error(str(e))
|
|
488 return config
|
|
489
|
|
490 def parse_args(self, *args, **kw):
|
|
491 """"""
|
|
492
|
|
493 # parse command line options
|
|
494 self.parsed = dict()
|
|
495 options = argparse.ArgumentParser.parse_args(self, *args, **kw)
|
|
496
|
|
497 # get CLI configuration options
|
|
498 cli_config = dict([(key, value) for key, value in options.__dict__.items()
|
|
499 if key in self.option_dict and key in self.parsed])
|
|
500
|
|
501 # deserialize configuration
|
|
502 config = self.read_configuration_files(options)
|
|
503 config.append(cli_config)
|
|
504
|
|
505 missingvalues = None
|
|
506 try:
|
|
507 # generate configuration
|
|
508 self(*config)
|
|
509 except MissingValueException, missingvalues:
|
|
510 # errors are handled below
|
|
511 pass
|
|
512
|
|
513 # dump configuration
|
|
514 self.dump(options, missingvalues)
|
|
515
|
|
516 # update options from config
|
|
517 options.__dict__.update(self.config)
|
|
518
|
|
519 # return parsed arguments
|
|
520 return options
|
|
521
|
|
522 def dump(self, options, missingvalues):
|
|
523 """dump configuration, if specified"""
|
|
524
|
|
525 if missingvalues:
|
|
526 self.error(str(missingvalues))
|
|
527
|
|
528 dump = getattr(options, 'dump')
|
|
529 if dump:
|
|
530 # TODO: have a way of specifying format other than filename
|
|
531 self.serialize(dump)
|
|
532
|
|
533
|
|
534 ### serialization/deserialization
|
|
535
|
|
536 def formats(self):
|
|
537 """formats for deserialization"""
|
|
538 retval = []
|
|
539 for provider in self.configuration_providers:
|
|
540 if provider.extensions and hasattr(provider, 'write'):
|
|
541 retval.append(provider.extensions[0])
|
|
542 return retval
|
|
543
|
|
544 def configuration_provider(self, format):
|
|
545 """configuration provider guess for a given filename"""
|
|
546 for provider in self.configuration_providers:
|
|
547 if format in provider.extensions:
|
|
548 return provider
|
|
549
|
|
550 def filename2format(self, filename):
|
|
551 extension = os.path.splitext(filename)[-1]
|
|
552 return extension.lstrip('.') or None
|
|
553
|
|
554 def serialize(self, filename, format=None, full=False):
|
|
555 """
|
|
556 serialize configuration to a file
|
|
557 - filename: path of file to serialize to
|
|
558 - format: format of configuration provider
|
|
559 - full: whether to serialize non-set optional strings [TODO]
|
|
560 """
|
|
561 # TODO: allow file object vs file name
|
|
562
|
|
563 if not format:
|
|
564 format = self.filename2format(filename)
|
|
565 if not format:
|
|
566 raise Exception('Please specify a format')
|
|
567 # TODO: more specific exception type
|
|
568
|
|
569 provider = self.configuration_provider(format)
|
|
570 if not provider:
|
|
571 raise Exception("Provider not found for format: %s" % format)
|
|
572
|
|
573 config = copy.deepcopy(self.config)
|
|
574
|
|
575 provider.write(config, filename)
|
|
576
|
|
577 def deserialize(self, filename, format=None):
|
|
578 """load configuration from a file"""
|
|
579 # TODO: allow file object vs file name
|
|
580
|
|
581 assert os.path.exists(filename)
|
|
582
|
|
583 # get the format
|
|
584 format = format or self.filename2format(filename)
|
|
585
|
|
586 # get the providers in some sensible order
|
|
587 providers = self.configuration_providers[:]
|
|
588 if format:
|
|
589 providers.sort(key=lambda x: int(format in x.extensions), reverse=True)
|
|
590
|
|
591 # deserialize the data
|
|
592 for provider in providers:
|
|
593 try:
|
|
594 return provider.read(filename)
|
|
595 except:
|
|
596 continue
|
|
597 else:
|
|
598 raise ConfigurationProviderException("Could not load {}".format(filename))
|
|
599
|
|
600
|
|
601 class UserConfiguration(Configuration):
|
|
602 """`Configuration` class driven by a config file in user-space"""
|
|
603
|
|
604 # configuration items to interpolate as paths
|
|
605 # TODO: integrate with above BaseCLI
|
|
606 paths = []
|
|
607
|
|
608 def __init__(self, config=None, load='--config', **kwargs):
|
|
609
|
|
610 # default configuration file
|
|
611 self.config_name = config or '.' + os.path.splitext(sys.argv[0])[0]
|
|
612 self.default_config_file = os.path.join('~', self.config_name)
|
|
613 self.default_config_file_path = os.path.expanduser(self.default_config_file)
|
|
614 if os.path.exists(self.default_config_file_path):
|
|
615 self.load_help += ' [DEFAULT: %s]' % self.default_config_file
|
|
616
|
|
617 Configuration.__init__(self, load=load, **kwargs)
|
|
618
|
|
619 def validate(self):
|
|
620 Configuration.validate(self)
|
|
621 for path in self.paths:
|
|
622 self.config[path] = os.path.expanduser(self.config[path])
|
|
623
|
|
624 def configuration_files(self, options, args):
|
|
625 configuration_files = Configuration.configuration_files(self, options, args)
|
|
626 if not configuration_files:
|
|
627 # load user config only if no config provided
|
|
628 if os.path.exists(self.default_config_file_path):
|
|
629 configuration_files = [self.default_config_file_path]
|
|
630 return configuration_files
|
|
631
|
|
632 def load_configuration_file(self, filename):
|
|
633 config = Configuration.load_configuration_file(self, filename)
|
|
634
|
|
635 # ignore options that we don't care about
|
|
636 config = dict([(key, value) for key, value in config.items()
|
|
637 if key in self.option_dict])
|
|
638
|
|
639 return config
|