Mercurial > hg > TextShaper
view textshaper/split.py @ 53:3691ffa84a3a
add to setup
author | Jeff Hammel <k0scist@gmail.com> |
---|---|
date | Sun, 17 May 2015 08:53:11 -0700 |
parents | 8d8c1ac0e8e1 |
children | 1d755747e67a |
line wrap: on
line source
#!/usr/bin/env python """ split paragraphs, sentences, etc """ # imports import argparse import re import string import sys def findall(_string, sub): """find all occurances of `sub` in _string""" retval = [] index = 0 while True: try: index = _string.index(sub, index) retval.append(index) index += len(sub) except ValueError: return retval def indices(text, values): """ returns ordered list of 2-tuples: (index, value) """ locations = {value: findall(text, value) for value in values} indices = [] for key, values in locations.items(): indices.extend([(value, key) for value in values]) return sorted(indices, key=lambda x: x[0]) def split_sentences(text, ends='.?!'): """split a text into sentences""" text = text.strip() sentences = [] _indices = indices(text, ends) begin = 0 for index, value in _indices: sentence = text[begin:index] sentence += value sentence.strip() begin = index if sentence: sentences.append(sentence) import pdb; pdb.set_trace() def split_paragraphs(text): lines = [line.strip() for line in text.strip().splitlines()] lines = [line if line else '\n' for line in lines] text = ' '.join(lines).strip() paragraphs = [' '.join(p) for p in text.split('\n')] return paragraphs def main(args=sys.argv[1:]): """CLI""" # parse command line arguments parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('file', nargs='?', type=argparse.FileType('r'), default=sys.stdin) options = parser.parse_args(args) # preprocess text text = options.file.read().strip() text = ' '.join(text.split()) # paragraphs = split_paragraphs(text) # find all sentences ends = '.?!' sentences = split_sentences(text, ends) # display for sentence in sentences: print (sentence) if __name__ == '__main__': main()