Mercurial > hg > TextShaper
annotate textshaper/split.py @ 54:1d755747e67a
almost there
| author | Jeff Hammel <k0scist@gmail.com> |
|---|---|
| date | Sun, 17 May 2015 09:11:30 -0700 |
| parents | 8d8c1ac0e8e1 |
| children | 4e2190495d50 |
| rev | line source |
|---|---|
| 45 | 1 #!/usr/bin/env python |
| 2 | |
| 3 """ | |
| 4 split paragraphs, sentences, etc | |
| 5 """ | |
| 6 | |
| 7 # imports | |
| 8 import argparse | |
| 54 | 9 import csv |
| 46 | 10 import string |
| 45 | 11 import sys |
| 12 | |
| 46 | 13 |
| 50 | 14 def findall(_string, sub): |
| 46 | 15 """find all occurances of `sub` in _string""" |
| 16 | |
| 17 retval = [] | |
| 18 index = 0 | |
| 19 while True: | |
| 20 try: | |
| 21 index = _string.index(sub, index) | |
| 22 retval.append(index) | |
| 50 | 23 index += len(sub) |
| 46 | 24 except ValueError: |
| 25 return retval | |
| 26 | |
| 51 | 27 def indices(text, values): |
| 50 | 28 """ |
| 29 returns ordered list of 2-tuples: | |
| 30 (index, value) | |
| 31 """ | |
| 51 | 32 locations = {value: findall(text, value) for value in values} |
| 33 indices = [] | |
| 34 for key, values in locations.items(): | |
| 35 indices.extend([(value, key) for value in values]) | |
| 36 return sorted(indices, key=lambda x: x[0]) | |
| 50 | 37 |
| 38 def split_sentences(text, ends='.?!'): | |
| 39 """split a text into sentences""" | |
| 46 | 40 |
|
52
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
41 text = text.strip() |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
42 sentences = [] |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
43 _indices = indices(text, ends) |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
44 |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
45 begin = 0 |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
46 for index, value in _indices: |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
47 sentence = text[begin:index] |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
48 sentence += value |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
49 sentence.strip() |
| 54 | 50 begin = index + len(value) |
|
52
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
51 if sentence: |
|
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
52 sentences.append(sentence) |
| 54 | 53 # add the trailing bits, if they exist |
| 54 sentence = text[begin:].strip() | |
| 55 if sentence: | |
| 56 sentences.append(sentence) | |
| 57 # shouldn't need to do this | |
| 58 sentences = [sentence.strip() for sentence in sentences] | |
| 59 return sentences | |
| 60 | |
|
52
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
61 |
| 45 | 62 def split_paragraphs(text): |
| 63 | |
| 64 lines = [line.strip() for line in text.strip().splitlines()] | |
| 65 lines = [line if line else '\n' | |
| 66 for line in lines] | |
| 67 text = ' '.join(lines).strip() | |
| 68 paragraphs = [' '.join(p) for p in text.split('\n')] | |
| 69 return paragraphs | |
| 70 | |
| 54 | 71 def words(text): |
| 72 """return the alphanumeric words in a sentence""" | |
| 73 words = text.strip().split() | |
| 74 return [word for word in words] | |
| 75 | |
| 45 | 76 def main(args=sys.argv[1:]): |
| 77 """CLI""" | |
| 78 | |
| 79 # parse command line arguments | |
| 80 parser = argparse.ArgumentParser(description=__doc__) | |
| 81 parser.add_argument('file', nargs='?', type=argparse.FileType('r'), default=sys.stdin) | |
| 54 | 82 parser.add_argument('-n', '--number', dest='number', |
| 83 action='store_true', default=False, | |
| 84 help="number the sentences (CSV)") | |
| 85 parser.add_argument('-o', '--output', dest='output', | |
| 86 type=argparse.FileType('w'), default=sys.stdout, | |
| 87 help="file to output to, or stdout by default") | |
| 45 | 88 options = parser.parse_args(args) |
| 89 | |
| 46 | 90 # preprocess text |
| 45 | 91 text = options.file.read().strip() |
| 92 text = ' '.join(text.split()) | |
| 48 | 93 # paragraphs = split_paragraphs(text) |
| 45 | 94 |
|
52
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
95 # find all sentences |
| 48 | 96 ends = '.?!' |
|
52
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
97 sentences = split_sentences(text, ends) |
| 45 | 98 |
|
52
8d8c1ac0e8e1
add a test text and wire some things up
Jeff Hammel <k0scist@gmail.com>
parents:
51
diff
changeset
|
99 # display |
| 54 | 100 if options.number: |
| 101 writer = csv.writer(options.output) | |
| 102 else: | |
| 103 for sentence in sentences: | |
| 104 options.output.write(sentence + '\n') | |
| 45 | 105 |
| 106 if __name__ == '__main__': | |
| 107 main() |