Mercurial > hg > toolbox

annotate toolbox/model.py @ 13:2d05ad362637

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
py3
author Jeff Hammel <k0scist@gmail.com>
date Tue, 24 Nov 2020 13:11:51 -0800
parents c280a43ac442
children 8e13c2f6c2d7
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
1 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
2 models for toolbox
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
3 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
4
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
5 import couchdb
13
Jeff Hammel <k0scist@gmail.com>
parents: 12
diff changeset
6 import json
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
7 import os
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
8 import pyes
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
9 import sys
13
Jeff Hammel <k0scist@gmail.com>
parents: 12
diff changeset
10
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
11 from copy import deepcopy
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
12 from time import time
13
Jeff Hammel <k0scist@gmail.com>
parents: 12
diff changeset
13 from .search import WhooshSearch
Jeff Hammel <k0scist@gmail.com>
parents: 12
diff changeset
14 from .util import str2filename
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
15
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
16 # TODO: types of fields:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
17 # - string: a single string: {'type': 'string', 'name': 'name', 'required': True}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
18 # - field: a list of strings: {'type': 'field', 'name', 'usage'}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
19 # - dict: a subclassifier: {'type': '???', 'name': 'url', 'required': True}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
20 # - computed values, such as modified
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
21
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
22 class ProjectsModel(object):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
23 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
24 abstract base class for toolbox tools
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
25 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
26
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
27 def __init__(self, fields=None, required=('name', 'description', 'url'),
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
28 whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
29 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
30 - fields : list of fields to use, or None to calculate dynamically
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
31 - required : required data (strings)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
32 - whoosh_index : directory to keep whoosh index in
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
33 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
34 self.required = set(required)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
35
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
36 # reserved fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
37 self.reserved = self.required.copy()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
38 self.reserved.update(['modified']) # last modified, a computed value
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
39 self.search = WhooshSearch(whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
40
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
41 # classifier fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
42 self._fields = fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
43 self.field_set = set(fields or ())
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
44
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
45 def update_search(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
46 """update the search index"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
47 assert self.required.issubset(project.keys()) # XXX should go elsewhere
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
48 fields = dict([(field, project[field])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
49 for field in self.fields()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
50 if field in project])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
51
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
52 # keys must be strings, not unicode, on some systems
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
53 f = dict([(str(i), j) for i, j in fields.items()])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
54
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
55 self.search.update(name=project['name'], description=project['description'], **f)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
56
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
57 def fields(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
58 """what fields does the model support?"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
59 if self._fields is not None:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
60 return self._fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
61 return list(self.field_set)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
62
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
63 def projects(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
64 """list of all projects"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
65 return [i['name'] for i in self.get()]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
66
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
67 def export(self, other):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
68 """export the current model to another model instance"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
69 for project in self.get():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
70 other.update(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
71
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
72 def rename_field_value(self, field, from_value, to_value):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
73 projects = self.get(None, **{field: from_value})
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
74 for project in projects:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
75 project[field].remove(from_value)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
76 project[field].append(to_value)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
77 self.update(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
78
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
79 ### implementor methods
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
80
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
81 def update(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
82 """update a project"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
83 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
84
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
85 def get(self, search=None, **query):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
86 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
87 get a list of projects matching a query
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
88 the query should be key, value pairs to match;
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
89 if the value is single, it should be a string;
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
90 if the value is multiple, it should be a set which will be
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
91 ANDed together
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
92 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
93 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
94
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
95 def project(self, name):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
96 """get a project of a particular name, or None if there is none"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
97 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
98
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
99 def field_query(self, field):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
100 """get projects according to a particular field, or None"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
101 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
102
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
103 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
104 raise NotImplementedError
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
105
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
106
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
107 class MemoryCache(ProjectsModel):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
108 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
109 sample implementation keeping everything in memory
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
110 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
111
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
112 def __init__(self, fields=None, whoosh_index=None):
11
Jeff Hammel <k0scist@gmail.com>
parents: 0
diff changeset
113
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
114 ProjectsModel.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
115
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
116 # indices
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
117 self._projects = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
118 self.index = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
119 self.load()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
120
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
121 def update(self, project, load=False):
11
Jeff Hammel <k0scist@gmail.com>
parents: 0
diff changeset
122
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
123 if project['name'] in self._projects and project == self._projects[project['name']]:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
124 return # nothing to do
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
125 if not load:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
126 project['modified'] = time()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
127 if self._fields is None:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
128 fields = [i for i in project if i not in self.reserved]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
129 self.field_set.update(fields)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
130 else:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
131 fields = self._fields
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
132 for field in fields:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
133 for key, _set in self.index.get(field, {}).items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
134 _set.discard(project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
135 if not _set:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
136 self.index[field].pop(key)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
137 if field not in project:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
138 continue
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
139 project[field] = list(set([i.strip() for i in project[field] if i.strip()]))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
140 index = self.index.setdefault(field, {})
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
141 values = project[field]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
142 if isinstance(values, basestring):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
143 values = [values]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
144 for value in values:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
145 index.setdefault(value, set()).update([project['name']])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
146 self._projects[project['name']] = deepcopy(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
147 self.update_search(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
148 if not load:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
149 self.save(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
150
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
151 def get(self, search=None, **query):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
152 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
153 - search: text search
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
154 - query: fields to match
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
155 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
156 order = None
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
157 if search:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
158 results = self.search(search)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
159 order = dict([(j,i) for i,j in enumerate(results)])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
160 else:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
161 results = self._projects.keys()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
162 results = set(results)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
163 for key, values in query.items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
164 if isinstance(values, basestring):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
165 values = [values]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
166 for value in values:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
167 results.intersection_update(self.index.get(key, {}).get(value, set()))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
168 if order:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
169 # preserve search order
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
170 results = sorted(list(results), key=lambda x: order[x])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
171 return [deepcopy(self._projects[project]) for project in results]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
172
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
173
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
174 def project(self, name):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
175 if name in self._projects:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
176 return deepcopy(self._projects[name])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
177
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
178 def field_query(self, field):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
179 if field in self.index:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
180 return deepcopy(self.index.get(field))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
181
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
182 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
183 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
184 delete a project
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
185 - project : name of the project
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
186 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
187 if project not in self._projects:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
188 return
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
189 del self._projects[project]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
190 for field, classifiers in self.index.items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
191 for key, values in classifiers.items():
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
192 classifiers[key].discard(project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
193 if not classifiers[key]:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
194 del classifiers[key]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
195 self.search.delete(project)
11
Jeff Hammel <k0scist@gmail.com>
parents: 0
diff changeset
196
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
197 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
198 """for subclasses; in memory, load nothing"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
199
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
200 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
201 """for subclasses; in memory, save nothing"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
202
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
203
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
204 class FileCache(MemoryCache):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
205 """save in JSON blob directory"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
206
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
207 def __init__(self, directory, fields=None, whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
208 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
209 - directory: directory of .json tool files
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
210 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
211 # JSON blob directory
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
212 if not os.path.exists(directory):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
213 os.makedirs(directory)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
214 assert os.path.isdir(directory)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
215 self.directory = directory
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
216
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
217 self.files = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
218 MemoryCache.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
219
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
220 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
221 MemoryCache.delete(self, project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
222 os.remove(os.path.join(self.directory, self.files.pop(project)))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
223
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
224 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
225 """load JSON from the directory"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
226 for i in os.listdir(self.directory):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
227 if not i.endswith('.json'):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
228 continue
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
229 filename = os.path.join(self.directory, i)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
230 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
231 project = json.loads(file(filename).read())
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
232 except:
11
Jeff Hammel <k0scist@gmail.com>
parents: 0
diff changeset
233 print('File: {}'.format(i))
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
234 raise
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
235 self.files[project['name']] = i
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
236 self.update(project, load='modified' in project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
237
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
238 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
239
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
240 filename = self.files.get(project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
241 if not filename:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
242 filename = str2filename(project['name']) + '.json'
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
243 filename = filename.encode('ascii', 'ignore')
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
244 filename = os.path.join(self.directory, filename)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
245 try:
12
Jeff Hammel <k0scist@gmail.com>
parents: 11
diff changeset
246 with open(filename, 'w') as f:
Jeff Hammel <k0scist@gmail.com>
parents: 11
diff changeset
247 f.write(json.dumps(project))
Jeff Hammel <k0scist@gmail.com>
parents: 11
diff changeset
248 except Exception as e:
Jeff Hammel <k0scist@gmail.com>
parents: 11
diff changeset
249 print('{} {}'.format(filename, repr(filename)))
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
250 raise
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
251
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
252
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
253 class ElasticSearchCache(MemoryCache):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
254 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
255 store json in ElasticSearch
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
256 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
257
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
258 def __init__(self,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
259 server="localhost:9200",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
260 es_index="toolbox",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
261 doc_type="projects",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
262 fields=None,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
263 whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
264 self.es_index = es_index
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
265 self.doc_type = doc_type
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
266
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
267 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
268 self.es = pyes.ES([server])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
269 self.es.create_index(self.es_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
270 except pyes.urllib3.connectionpool.MaxRetryError:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
271 raise Exception("Could not connect to ES instance")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
272 except pyes.exceptions.ElasticSearchException:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
273 # this just means the index already exists
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
274 pass
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
275 MemoryCache.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
276
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
277 def es_query(self, query):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
278 """make an ElasticSearch query and return the results"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
279 search = pyes.Search(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
280 results = self.es.search(query=search,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
281 indexes=[self.es_index],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
282 doc_types=[self.doc_type],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
283 size=0)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
284
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
285 # the first query is just used to determine the size of the set
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
286 if not 'hits' in results and not 'total' in results['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
287 raise Exception("bad ES response %s" % json.dumps(results))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
288 total = results['hits']['total']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
289
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
290 # repeat the query to retrieve the entire set
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
291 results = self.es.search(query=search,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
292 indexes=[self.es_index],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
293 doc_types=[self.doc_type],
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
294 size=total)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
295
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
296 if not 'hits' in results and not 'hits' in results['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
297 raise Exception("bad ES response %s" % json.dumps(results))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
298
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
299 return results
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
300
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
301 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
302 """load all json documents from ES:toolbox/projects"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
303 query = pyes.MatchAllQuery()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
304 results = self.es_query(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
305
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
306 for hit in results['hits']['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
307 self.update(hit['_source'], True)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
308
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
309 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
310 query = pyes.FieldQuery()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
311 query.add('name', project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
312 results = self.es_query(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
313
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
314 # If there is an existing records in ES with the same
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
315 # project name, update that record. Otherwise create a new record.
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
316 id = None
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
317 if results['hits']['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
318 id = results['hits']['hits'][0]['_id']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
319
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
320 self.es.index(project, self.es_index, self.doc_type, id)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
321
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
322 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
323 MemoryCache.delete(self, project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
324 query = pyes.FieldQuery()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
325 query.add('name', project['name'])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
326 results = self.es_query(query)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
327 if results['hits']['hits']:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
328 id = results['hits']['hits'][0]['_id']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
329 self.es.delete(self.es_index, self.doc_type, id)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
330
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
331
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
332 class CouchCache(MemoryCache):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
333 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
334 store json files in couchdb
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
335 """
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
336
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
337 def __init__(self,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
338 server="http://127.0.0.1:5984",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
339 dbname="toolbox",
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
340 fields=None,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
341 whoosh_index=None):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
342
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
343 # TODO: check if server is running
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
344 couchserver = couchdb.Server(server)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
345 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
346 self.db = couchserver[dbname]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
347 except couchdb.ResourceNotFound: # XXX should not be a blanket except!
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
348 self.db = couchserver.create(dbname)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
349 except:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
350 raise Exception("Could not connect to couch instance. Make sure that you have couch running at %s and that you have database create priveleges if '%s' does not exist" % (server, dbname))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
351 MemoryCache.__init__(self, fields=fields, whoosh_index=whoosh_index)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
352
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
353 def load(self):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
354 """load JSON objects from CouchDB docs"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
355 for id in self.db:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
356 doc = self.db[id]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
357 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
358 project = doc['project']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
359 except KeyError:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
360 continue # it's prob a design doc
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
361 self.update(project, load=True)
11
Jeff Hammel <k0scist@gmail.com>
parents: 0
diff changeset
362
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
363 def save(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
364 name = project['name']
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
365 try:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
366 updated = self.db[name]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
367 except:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
368 updated = {}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
369 updated['project'] = project
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
370 self.db[name] = updated
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
371
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
372 def delete(self, project):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
373 MemoryCache.delete(self, project)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
374 del self.db[project]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
375
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
376 # directory of available models
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
377 models = {'memory_cache': MemoryCache,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
378 'file_cache': FileCache,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
379 'couch': CouchCache,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
380 'es': ElasticSearchCache}
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
381
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
382 def convert(args=sys.argv[1:]):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
383 """CLI front-end for model conversion"""
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
384 from optparse import OptionParser
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
385 usage = '%prog [global-options] from_model [options] to_model [options]'
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
386 description = "export data from one model to another"
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
387 parser = OptionParser(usage=usage, description=description)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
388 parser.disable_interspersed_args()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
389 parser.add_option('-l', '--list-models', dest='list_models',
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
390 action='store_true', default=False,
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
391 help="list available models")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
392 parser.add_option('-a', '--list-args', dest='list_args',
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
393 metavar='MODEL',
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
394 help="list arguments for a model")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
395
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
396 options, args = parser.parse_args(args)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
397
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
398 # process global options
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
399 if options.list_models:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
400 for name in sorted(models.keys()):
12
Jeff Hammel <k0scist@gmail.com>
parents: 11
diff changeset
401 print(name)
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
402 parser.exit()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
403 if options.list_args:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
404 if not options.list_args in models:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
405 parser.error("Model '%s' not found. (Choose from: %s)" % (options.list_args, models.keys()))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
406 ctor = models[options.list_args].__init__
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
407 import inspect
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
408 argspec = inspect.getargspec(ctor)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
409 defaults = [[i, None] for i in argspec.args[1:]] # ignore self
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
410 for index, value in enumerate(reversed(argspec.defaults), 1):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
411 defaults[-index][-1] = value
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
412 defaults = [[i,j] for i, j in defaults if i != 'fields']
12
Jeff Hammel <k0scist@gmail.com>
parents: 11
diff changeset
413 print('%s arguments:' % options.list_args)
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
414 for arg, value in defaults:
12
Jeff Hammel <k0scist@gmail.com>
parents: 11
diff changeset
415 print(' -%s %s' % (arg, value or ''))
0
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
416 parser.exit()
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
417
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
418 # parse models and their ctor args
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
419 sects = []
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
420 _models = []
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
421 for arg in args:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
422 if arg.startswith('-'):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
423 sects[-1].append(arg)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
424 else:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
425 _models.append(arg)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
426 sects.append([])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
427
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
428 # check models
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
429 if len(_models) != 2:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
430 parser.error("Please provide two models. (You gave: %s)" % _models)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
431 if not set(_models).issubset(models):
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
432 parser.error("Please use these models: %s (You gave: %s)" % (models, _models))
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
433
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
434 sects = [ [i.lstrip('-') for i in sect ] for sect in sects ]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
435
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
436 # require an equals sign
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
437 # XXX hacky but much easier to parse
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
438 if [ True for sect in sects
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
439 if [i for i in sect if '=' not in i] ]:
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
440 parser.error("All arguments must be `key=value`")
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
441 sects = [dict([i.split('=', 1) for i in sect]) for sect in sects]
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
442
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
443 # instantiate models
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
444 from_model = models[_models[0]](**sects[0])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
445 to_model = models[_models[1]](**sects[1])
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
446
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
447 # convert the data
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
448 from_model.export(to_model)
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
449
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
450 if __name__ == '__main__':
b0942f44413f import from git://github.com/mozilla/toolbox.git
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
451 convert()