|
45
|
1 #!/usr/bin/env python
|
|
|
2
|
|
|
3 """
|
|
|
4 split paragraphs, sentences, etc
|
|
|
5 """
|
|
|
6
|
|
|
7 # imports
|
|
|
8 import argparse
|
|
46
|
9 import re
|
|
|
10 import string
|
|
45
|
11 import sys
|
|
|
12
|
|
46
|
13
|
|
50
|
14 def findall(_string, sub):
|
|
46
|
15 """find all occurances of `sub` in _string"""
|
|
|
16
|
|
|
17 retval = []
|
|
|
18 index = 0
|
|
|
19 while True:
|
|
|
20 try:
|
|
|
21 index = _string.index(sub, index)
|
|
|
22 retval.append(index)
|
|
50
|
23 index += len(sub)
|
|
46
|
24 except ValueError:
|
|
|
25 return retval
|
|
|
26
|
|
51
|
27 def indices(text, values):
|
|
50
|
28 """
|
|
|
29 returns ordered list of 2-tuples:
|
|
|
30 (index, value)
|
|
|
31 """
|
|
51
|
32 locations = {value: findall(text, value) for value in values}
|
|
|
33 indices = []
|
|
|
34 for key, values in locations.items():
|
|
|
35 indices.extend([(value, key) for value in values])
|
|
|
36 return sorted(indices, key=lambda x: x[0])
|
|
50
|
37
|
|
|
38 def split_sentences(text, ends='.?!'):
|
|
|
39 """split a text into sentences"""
|
|
46
|
40
|
|
45
|
41 def split_paragraphs(text):
|
|
|
42
|
|
|
43 lines = [line.strip() for line in text.strip().splitlines()]
|
|
|
44 lines = [line if line else '\n'
|
|
|
45 for line in lines]
|
|
|
46 text = ' '.join(lines).strip()
|
|
|
47 paragraphs = [' '.join(p) for p in text.split('\n')]
|
|
|
48 return paragraphs
|
|
|
49
|
|
|
50 def main(args=sys.argv[1:]):
|
|
|
51 """CLI"""
|
|
|
52
|
|
|
53 # parse command line arguments
|
|
|
54 parser = argparse.ArgumentParser(description=__doc__)
|
|
|
55 parser.add_argument('file', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
|
|
|
56 options = parser.parse_args(args)
|
|
|
57
|
|
46
|
58 # preprocess text
|
|
45
|
59 text = options.file.read().strip()
|
|
|
60 text = ' '.join(text.split())
|
|
48
|
61 # paragraphs = split_paragraphs(text)
|
|
45
|
62
|
|
48
|
63 ends = '.?!'
|
|
45
|
64
|
|
48
|
65 # find all ending punctuation
|
|
51
|
66
|
|
48
|
67
|
|
45
|
68
|
|
|
69 if __name__ == '__main__':
|
|
|
70 main()
|
