annotate textshaper/split.py @ 46:7e63ca061b6c

start findall function
author Jeff Hammel <k0scist@gmail.com>
date Sat, 16 May 2015 18:53:53 -0700
parents ccbdc00d4f0a
children 03ce88daa98d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
1 #!/usr/bin/env python
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
2
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
3 """
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
4 split paragraphs, sentences, etc
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
5 """
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
6
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
7 # imports
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
8 import argparse
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
9 import re
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
10 import string
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
11 import sys
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
12
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
13
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
14 def findall(sub, _string):
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
15 """find all occurances of `sub` in _string"""
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
16
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
17 retval = []
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
18 index = 0
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
19 while True:
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
20 try:
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
21 index = _string.index(sub, index)
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
22 retval.append(index)
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
23 index += 1
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
24 except ValueError:
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
25 return retval
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
26
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
27
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
28 def split_paragraphs(text):
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
29
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
30 lines = [line.strip() for line in text.strip().splitlines()]
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
31 lines = [line if line else '\n'
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
32 for line in lines]
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
33 text = ' '.join(lines).strip()
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
34 paragraphs = [' '.join(p) for p in text.split('\n')]
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
35 return paragraphs
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
36
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
37 def main(args=sys.argv[1:]):
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
38 """CLI"""
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
39
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
40 # parse command line arguments
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
41 parser = argparse.ArgumentParser(description=__doc__)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
42 parser.add_argument('file', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
43 options = parser.parse_args(args)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
44
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
45 # preprocess text
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
46 text = options.file.read().strip()
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
47 text = ' '.join(text.split())
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
48 # paragraphs = split_paragraphs(text)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
49
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
50 ends = '.?!'
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
51
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
52 for end in ends:
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
53 # for paragraph in paragraphs:
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
54 # print (paragraph)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
55
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
56 if __name__ == '__main__':
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
57 main()