Mercurial > hg > TextShaper
view textshaper/split.py @ 51:c3b69728f291
finding indices now works
author | Jeff Hammel <k0scist@gmail.com> |
---|---|
date | Sun, 17 May 2015 08:33:23 -0700 |
parents | 1284c99a94fa |
children | 8d8c1ac0e8e1 |
line wrap: on
line source
#!/usr/bin/env python """ split paragraphs, sentences, etc """ # imports import argparse import re import string import sys def findall(_string, sub): """find all occurances of `sub` in _string""" retval = [] index = 0 while True: try: index = _string.index(sub, index) retval.append(index) index += len(sub) except ValueError: return retval def indices(text, values): """ returns ordered list of 2-tuples: (index, value) """ locations = {value: findall(text, value) for value in values} indices = [] for key, values in locations.items(): indices.extend([(value, key) for value in values]) return sorted(indices, key=lambda x: x[0]) def split_sentences(text, ends='.?!'): """split a text into sentences""" def split_paragraphs(text): lines = [line.strip() for line in text.strip().splitlines()] lines = [line if line else '\n' for line in lines] text = ' '.join(lines).strip() paragraphs = [' '.join(p) for p in text.split('\n')] return paragraphs def main(args=sys.argv[1:]): """CLI""" # parse command line arguments parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('file', nargs='?', type=argparse.FileType('r'), default=sys.stdin) options = parser.parse_args(args) # preprocess text text = options.file.read().strip() text = ' '.join(text.split()) # paragraphs = split_paragraphs(text) ends = '.?!' # find all ending punctuation if __name__ == '__main__': main()