Mercurial > hg > Lemuriformes

annotate lemuriformes/log.py @ 15:0d1b8bb1d97b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SQL + data related functionality
author Jeff Hammel <k0scist@gmail.com>
date Sun, 10 Dec 2017 17:16:52 -0800
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
15
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
1 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
2 ElasticSearch-style logging
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
3 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
4
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
5 import json
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
6 import sys
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
7 import time
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
8 from .cli import ConfigurationParser
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
9
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
10 stdout = sys.stdout
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
11
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
12 try:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
13 # python 2
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
14 string = (str, unicode)
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
15 except NameError:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
16 # python 3
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
17 string = (str, )
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
18
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
19
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
20 def read_logfile(f):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
21 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
22 read a JSON-per-line log file's contents and return the value
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
23
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
24 f -- log file pointer or name
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
25 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
26
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
27 if isinstance(f, string):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
28 with open(f) as _f:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
29 return read_logfile(_f)
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
30 lines = f.read().strip().splitlines()
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
31 return [json.loads(line) for line in lines]
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
32
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
33
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
34 class ElasticLogger(object):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
35 """Elasticsearch-compatible log dispatcher"""
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
36
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
37 def __init__(self, logfilepath=None, **data):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
38 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
39 logfilepath -- path to logfile
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
40 data -- data to be included with each logging event
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
41 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
42
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
43 self.logfilepath = logfilepath
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
44 self.data = data
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
45
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
46 def write(self, f, **data):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
47 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
48 write JSON `data` to file-like object `f`
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
49 """
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
50
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
51 f.write(json.dumps(data, sort_keys=True) + '\n')
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
52 f.flush()
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
53
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
54 def __call__(self, message, **kw):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
55
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
56 # create log data JSON blob
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
57 now = time.time()
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
58 data = self.data.copy() # shallow copy
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
59 data.update({'time': now,
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
60 'message': message})
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
61 data.update(kw)
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
62
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
63 # log to stdout
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
64 self.write(stdout, **data)
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
65
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
66 if self.logfilepath:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
67 # log to a file
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
68 with open(self.logfilepath, 'a') as logfile:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
69 self.write(logfile, **data)
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
70
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
71
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
72 class ElasticLoggerParser(ConfigurationParser):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
73
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
74 def add_arguments(self):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
75 self.add_argument('-l', '--log', '--logfile', dest='logfile',
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
76 help="where to log events to in addition to stdout")
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
77 self.add_argument('--tag', dest='tags', nargs='+', default=(),
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
78 type=self.keyvalue, metavar="KEY=VALUE",
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
79 help="set of key, values to tag all log lines with")
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
80
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
81 def logger(self):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
82 """return elastic logger instance"""
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
83
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
84 assert self.options is not None
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
85 return ElasticLogger(self.options.logfile,
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
86 **dict(self.options.tags))
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
87
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
88
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
89 def main(args=sys.argv[1:]):
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
90 """example CLI program"""
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
91
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
92 # parse command line
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
93 parser = ElasticLoggerParser(description="my elastic diary")
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
94 options = parser.parse_args()
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
95
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
96 # example: timestamped diary
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
97 logger = parser.logger()
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
98
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
99 # main loop
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
100 try:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
101 while True:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
102 logger(raw_input())
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
103 except KeyboardInterrupt:
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
104 pass
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
105
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
106
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
107 if __name__ == '__main__':
0d1b8bb1d97b SQL + data related functionality
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
108 main()