Mercurial > hg > TextShaper

annotate textshaper/split.py @ 53:3691ffa84a3a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add to setup
author Jeff Hammel <k0scist@gmail.com>
date Sun, 17 May 2015 08:53:11 -0700
parents 8d8c1ac0e8e1
children 1d755747e67a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
1 #!/usr/bin/env python
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
2
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
3 """
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
4 split paragraphs, sentences, etc
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
5 """
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
6
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
7 # imports
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
8 import argparse
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
9 import re
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
10 import string
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
11 import sys
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
12
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
13
50
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
14 def findall(_string, sub):
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
15 """find all occurances of `sub` in _string"""
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
16
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
17 retval = []
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
18 index = 0
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
19 while True:
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
20 try:
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
21 index = _string.index(sub, index)
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
22 retval.append(index)
50
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
23 index += len(sub)
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
24 except ValueError:
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
25 return retval
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
26
51
c3b69728f291 finding indices now works
Jeff Hammel <k0scist@gmail.com>
parents: 50
diff changeset
27 def indices(text, values):
50
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
28 """
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
29 returns ordered list of 2-tuples:
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
30 (index, value)
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
31 """
51
c3b69728f291 finding indices now works
Jeff Hammel <k0scist@gmail.com>
parents: 50
diff changeset
32 locations = {value: findall(text, value) for value in values}
c3b69728f291 finding indices now works
Jeff Hammel <k0scist@gmail.com>
parents: 50
diff changeset
33 indices = []
c3b69728f291 finding indices now works
Jeff Hammel <k0scist@gmail.com>
parents: 50
diff changeset
34 for key, values in locations.items():
c3b69728f291 finding indices now works
Jeff Hammel <k0scist@gmail.com>
parents: 50
diff changeset
35 indices.extend([(value, key) for value in values])
c3b69728f291 finding indices now works
Jeff Hammel <k0scist@gmail.com>
parents: 50
diff changeset
36 return sorted(indices, key=lambda x: x[0])
50
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
37
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
38 def split_sentences(text, ends='.?!'):
1284c99a94fa stubbing
Jeff Hammel <k0scist@gmail.com>
parents: 48
diff changeset
39 """split a text into sentences"""
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
40
52
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
41 text = text.strip()
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
42 sentences = []
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
43 _indices = indices(text, ends)
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
44
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
45 begin = 0
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
46 for index, value in _indices:
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
47 sentence = text[begin:index]
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
48 sentence += value
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
49 sentence.strip()
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
50 begin = index
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
51 if sentence:
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
52 sentences.append(sentence)
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
53 import pdb; pdb.set_trace()
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
54
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
55 def split_paragraphs(text):
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
56
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
57 lines = [line.strip() for line in text.strip().splitlines()]
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
58 lines = [line if line else '\n'
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
59 for line in lines]
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
60 text = ' '.join(lines).strip()
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
61 paragraphs = [' '.join(p) for p in text.split('\n')]
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
62 return paragraphs
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
63
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
64 def main(args=sys.argv[1:]):
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
65 """CLI"""
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
66
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
67 # parse command line arguments
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
68 parser = argparse.ArgumentParser(description=__doc__)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
69 parser.add_argument('file', nargs='?', type=argparse.FileType('r'), default=sys.stdin)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
70 options = parser.parse_args(args)
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
71
46
7e63ca061b6c start findall function
Jeff Hammel <k0scist@gmail.com>
parents: 45
diff changeset
72 # preprocess text
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
73 text = options.file.read().strip()
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
74 text = ' '.join(text.split())
48
03ce88daa98d start test
Jeff Hammel <k0scist@gmail.com>
parents: 46
diff changeset
75 # paragraphs = split_paragraphs(text)
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
76
52
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
77 # find all sentences
48
03ce88daa98d start test
Jeff Hammel <k0scist@gmail.com>
parents: 46
diff changeset
78 ends = '.?!'
52
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
79 sentences = split_sentences(text, ends)
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
80
52
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
81 # display
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
82 for sentence in sentences:
8d8c1ac0e8e1 add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents: 51
diff changeset
83 print (sentence)
45
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
84
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
85 if __name__ == '__main__':
Jeff Hammel <k0scist@gmail.com>
parents:
diff changeset
86 main()