Mercurial > hg > toolbox

annotate toolbox/model.py @ 0:b0942f44413f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
import from git://github.com/mozilla/toolbox.git
author Jeff Hammel <k0scist@gmail.com>
date Sun, 11 May 2014 09:15:35 -0700
parents
children 1c6dbeb4ddab
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
1 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
2 models for toolbox
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
3 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
4
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
5 import couchdb
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
6 import os
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
7 import pyes
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
8 import sys
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
9 from copy import deepcopy
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
10 from search import WhooshSearch
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
11 from time import time
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
12 from util import str2filename
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
13
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
14 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
15 import json
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
16 except ImportError:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
17 import simplejson as json
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
18
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
19 # TODO: types of fields:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
20 # - string: a single string: {'type': 'string', 'name': 'name', 'required': True}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
21 # - field: a list of strings: {'type': 'field', 'name', 'usage'}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
22 # - dict: a subclassifier: {'type': '???', 'name': 'url', 'required': True}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
23 # - computed values, such as modified
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
24
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
25 class ProjectsModel(object):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
26 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
27 abstract base class for toolbox tools
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
28 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
29
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
30 def __init__(self, fields=None, required=('name', 'description', 'url'),
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
31 whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
32 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
33 - fields : list of fields to use, or None to calculate dynamically
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
34 - required : required data (strings)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
35 - whoosh_index : directory to keep whoosh index in
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
36 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
37 self.required = set(required)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
38
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
39 # reserved fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
40 self.reserved = self.required.copy()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
41 self.reserved.update(['modified']) # last modified, a computed value
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
42 self.search = WhooshSearch(whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
43
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
44 # classifier fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
45 self._fields = fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
46 self.field_set = set(fields or ())
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
47
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
48 def update_search(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
49 """update the search index"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
50 assert self.required.issubset(project.keys()) # XXX should go elsewhere
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
51 fields = dict([(field, project[field])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
52 for field in self.fields()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
53 if field in project])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
54
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
55 # keys must be strings, not unicode, on some systems
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
56 f = dict([(str(i), j) for i, j in fields.items()])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
57
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
58 self.search.update(name=project['name'], description=project['description'], **f)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
59
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
60 def fields(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
61 """what fields does the model support?"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
62 if self._fields is not None:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
63 return self._fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
64 return list(self.field_set)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
65
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
66 def projects(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
67 """list of all projects"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
68 return [i['name'] for i in self.get()]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
69
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
70 def export(self, other):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
71 """export the current model to another model instance"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
72 for project in self.get():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
73 other.update(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
74
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
75 def rename_field_value(self, field, from_value, to_value):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
76 projects = self.get(None, **{field: from_value})
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
77 for project in projects:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
78 project[field].remove(from_value)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
79 project[field].append(to_value)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
80 self.update(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
81
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
82 ### implementor methods
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
83
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
84 def update(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
85 """update a project"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
86 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
87
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
88 def get(self, search=None, **query):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
89 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
90 get a list of projects matching a query
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
91 the query should be key, value pairs to match;
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
92 if the value is single, it should be a string;
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
93 if the value is multiple, it should be a set which will be
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
94 ANDed together
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
95 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
96 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
97
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
98 def project(self, name):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
99 """get a project of a particular name, or None if there is none"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
100 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
101
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
102 def field_query(self, field):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
103 """get projects according to a particular field, or None"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
104 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
105
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
106 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
107 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
108
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
109
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
110 class MemoryCache(ProjectsModel):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
111 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
112 sample implementation keeping everything in memory
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
113 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
114
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
115 def __init__(self, fields=None, whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
116
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
117 ProjectsModel.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
118
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
119 # indices
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
120 self._projects = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
121 self.index = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
122
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
123 self.load()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
124
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
125 def update(self, project, load=False):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
126
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
127 if project['name'] in self._projects and project == self._projects[project['name']]:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
128 return # nothing to do
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
129 if not load:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
130 project['modified'] = time()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
131 if self._fields is None:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
132 fields = [i for i in project if i not in self.reserved]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
133 self.field_set.update(fields)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
134 else:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
135 fields = self._fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
136 for field in fields:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
137 for key, _set in self.index.get(field, {}).items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
138 _set.discard(project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
139 if not _set:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
140 self.index[field].pop(key)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
141 if field not in project:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
142 continue
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
143 project[field] = list(set([i.strip() for i in project[field] if i.strip()]))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
144 index = self.index.setdefault(field, {})
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
145 values = project[field]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
146 if isinstance(values, basestring):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
147 values = [values]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
148 for value in values:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
149 index.setdefault(value, set()).update([project['name']])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
150 self._projects[project['name']] = deepcopy(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
151 self.update_search(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
152 if not load:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
153 self.save(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
154
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
155 def get(self, search=None, **query):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
156 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
157 - search: text search
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
158 - query: fields to match
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
159 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
160 order = None
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
161 if search:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
162 results = self.search(search)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
163 order = dict([(j,i) for i,j in enumerate(results)])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
164 else:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
165 results = self._projects.keys()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
166 results = set(results)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
167 for key, values in query.items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
168 if isinstance(values, basestring):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
169 values = [values]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
170 for value in values:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
171 results.intersection_update(self.index.get(key, {}).get(value, set()))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
172 if order:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
173 # preserve search order
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
174 results = sorted(list(results), key=lambda x: order[x])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
175 return [deepcopy(self._projects[project]) for project in results]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
176
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
177
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
178 def project(self, name):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
179 if name in self._projects:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
180 return deepcopy(self._projects[name])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
181
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
182 def field_query(self, field):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
183 if field in self.index:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
184 return deepcopy(self.index.get(field))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
185
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
186 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
187 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
188 delete a project
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
189 - project : name of the project
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
190 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
191 if project not in self._projects:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
192 return
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
193 del self._projects[project]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
194 for field, classifiers in self.index.items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
195 for key, values in classifiers.items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
196 classifiers[key].discard(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
197 if not classifiers[key]:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
198 del classifiers[key]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
199 self.search.delete(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
200
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
201 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
202 """for subclasses; in memory, load nothing"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
203
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
204 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
205 """for subclasses; in memory, save nothing"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
206
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
207
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
208 class FileCache(MemoryCache):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
209 """save in JSON blob directory"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
210
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
211 def __init__(self, directory, fields=None, whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
212 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
213 - directory: directory of .json tool files
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
214 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
215 # JSON blob directory
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
216 if not os.path.exists(directory):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
217 os.makedirs(directory)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
218 assert os.path.isdir(directory)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
219 self.directory = directory
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
220
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
221 self.files = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
222 MemoryCache.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
223
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
224 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
225 MemoryCache.delete(self, project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
226 os.remove(os.path.join(self.directory, self.files.pop(project)))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
227
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
228 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
229 """load JSON from the directory"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
230 for i in os.listdir(self.directory):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
231 if not i.endswith('.json'):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
232 continue
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
233 filename = os.path.join(self.directory, i)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
234 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
235 project = json.loads(file(filename).read())
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
236 except:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
237 print 'File: ' + i
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
238 raise
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
239 self.files[project['name']] = i
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
240 self.update(project, load='modified' in project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
241
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
242 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
243
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
244 filename = self.files.get(project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
245 if not filename:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
246 filename = str2filename(project['name']) + '.json'
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
247 filename = filename.encode('ascii', 'ignore')
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
248 filename = os.path.join(self.directory, filename)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
249 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
250 f = file(filename, 'w')
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
251 except Exception, e:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
252 print filename, repr(filename)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
253 raise
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
254 f.write(json.dumps(project))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
255 f.close()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
256
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
257
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
258 class ElasticSearchCache(MemoryCache):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
259 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
260 store json in ElasticSearch
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
261 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
262
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
263 def __init__(self,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
264 server="localhost:9200",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
265 es_index="toolbox",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
266 doc_type="projects",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
267 fields=None,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
268 whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
269 self.es_index = es_index
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
270 self.doc_type = doc_type
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
271
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
272 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
273 self.es = pyes.ES([server])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
274 self.es.create_index(self.es_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
275 except pyes.urllib3.connectionpool.MaxRetryError:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
276 raise Exception("Could not connect to ES instance")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
277 except pyes.exceptions.ElasticSearchException:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
278 # this just means the index already exists
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
279 pass
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
280 MemoryCache.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
281
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
282 def es_query(self, query):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
283 """make an ElasticSearch query and return the results"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
284 search = pyes.Search(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
285 results = self.es.search(query=search,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
286 indexes=[self.es_index],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
287 doc_types=[self.doc_type],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
288 size=0)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
289
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
290 # the first query is just used to determine the size of the set
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
291 if not 'hits' in results and not 'total' in results['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
292 raise Exception("bad ES response %s" % json.dumps(results))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
293 total = results['hits']['total']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
294
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
295 # repeat the query to retrieve the entire set
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
296 results = self.es.search(query=search,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
297 indexes=[self.es_index],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
298 doc_types=[self.doc_type],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
299 size=total)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
300
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
301 if not 'hits' in results and not 'hits' in results['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
302 raise Exception("bad ES response %s" % json.dumps(results))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
303
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
304 return results
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
305
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
306 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
307 """load all json documents from ES:toolbox/projects"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
308 query = pyes.MatchAllQuery()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
309 results = self.es_query(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
310
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
311 for hit in results['hits']['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
312 self.update(hit['_source'], True)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
313
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
314 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
315 query = pyes.FieldQuery()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
316 query.add('name', project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
317 results = self.es_query(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
318
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
319 # If there is an existing records in ES with the same
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
320 # project name, update that record. Otherwise create a new record.
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
321 id = None
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
322 if results['hits']['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
323 id = results['hits']['hits'][0]['_id']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
324
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
325 self.es.index(project, self.es_index, self.doc_type, id)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
326
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
327 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
328 MemoryCache.delete(self, project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
329 query = pyes.FieldQuery()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
330 query.add('name', project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
331 results = self.es_query(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
332 if results['hits']['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
333 id = results['hits']['hits'][0]['_id']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
334 self.es.delete(self.es_index, self.doc_type, id)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
335
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
336
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
337 class CouchCache(MemoryCache):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
338 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
339 store json files in couchdb
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
340 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
341
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
342 def __init__(self,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
343 server="http://127.0.0.1:5984",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
344 dbname="toolbox",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
345 fields=None,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
346 whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
347
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
348 # TODO: check if server is running
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
349 couchserver = couchdb.Server(server)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
350 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
351 self.db = couchserver[dbname]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
352 except couchdb.ResourceNotFound: # XXX should not be a blanket except!
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
353 self.db = couchserver.create(dbname)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
354 except:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
355 raise Exception("Could not connect to couch instance. Make sure that you have couch running at %s and that you have database create priveleges if '%s' does not exist" % (server, dbname))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
356 MemoryCache.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
357
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
358 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
359 """load JSON objects from CouchDB docs"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
360 for id in self.db:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
361 doc = self.db[id]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
362 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
363 project = doc['project']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
364 except KeyError:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
365 continue # it's prob a design doc
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
366 self.update(project, load=True)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
367
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
368 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
369 name = project['name']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
370 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
371 updated = self.db[name]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
372 except:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
373 updated = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
374 updated['project'] = project
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
375 self.db[name] = updated
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
376
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
377 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
378 MemoryCache.delete(self, project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
379 del self.db[project]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
380
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
381 # directory of available models
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
382 models = {'memory_cache': MemoryCache,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
383 'file_cache': FileCache,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
384 'couch': CouchCache,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
385 'es': ElasticSearchCache}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
386
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
387 def convert(args=sys.argv[1:]):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
388 """CLI front-end for model conversion"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
389 from optparse import OptionParser
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
390 usage = '%prog [global-options] from_model [options] to_model [options]'
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
391 description = "export data from one model to another"
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
392 parser = OptionParser(usage=usage, description=description)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
393 parser.disable_interspersed_args()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
394 parser.add_option('-l', '--list-models', dest='list_models',
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
395 action='store_true', default=False,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
396 help="list available models")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
397 parser.add_option('-a', '--list-args', dest='list_args',
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
398 metavar='MODEL',
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
399 help="list arguments for a model")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
400
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
401 options, args = parser.parse_args(args)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
402
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
403 # process global options
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
404 if options.list_models:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
405 for name in sorted(models.keys()):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
406 print name # could conceivably print docstring
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
407 parser.exit()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
408 if options.list_args:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
409 if not options.list_args in models:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
410 parser.error("Model '%s' not found. (Choose from: %s)" % (options.list_args, models.keys()))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
411 ctor = models[options.list_args].__init__
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
412 import inspect
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
413 argspec = inspect.getargspec(ctor)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
414 defaults = [[i, None] for i in argspec.args[1:]] # ignore self
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
415 for index, value in enumerate(reversed(argspec.defaults), 1):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
416 defaults[-index][-1] = value
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
417 defaults = [[i,j] for i, j in defaults if i != 'fields']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
418 print '%s arguments:' % options.list_args
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
419 for arg, value in defaults:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
420 print ' -%s %s' % (arg, value or '')
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
421 parser.exit()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
422
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
423 # parse models and their ctor args
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
424 sects = []
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
425 _models = []
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
426 for arg in args:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
427 if arg.startswith('-'):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
428 sects[-1].append(arg)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
429 else:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
430 _models.append(arg)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
431 sects.append([])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
432
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
433 # check models
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
434 if len(_models) != 2:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
435 parser.error("Please provide two models. (You gave: %s)" % _models)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
436 if not set(_models).issubset(models):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
437 parser.error("Please use these models: %s (You gave: %s)" % (models, _models))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
438
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
439 sects = [ [i.lstrip('-') for i in sect ] for sect in sects ]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
440
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
441 # require an equals sign
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
442 # XXX hacky but much easier to parse
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
443 if [ True for sect in sects
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
444 if [i for i in sect if '=' not in i] ]:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
445 parser.error("All arguments must be `key=value`")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
446 sects = [dict([i.split('=', 1) for i in sect]) for sect in sects]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
447
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
448 # instantiate models
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
449 from_model = models[_models[0]](**sects[0])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
450 to_model = models[_models[1]](**sects[1])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
451
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
452 # convert the data
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
453 from_model.export(to_model)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
454
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
455 if __name__ == '__main__':
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
456 convert()