Skip to content
Snippets Groups Projects
Commit 92689087 authored by vvye's avatar vvye
Browse files

Rename sentence summarization to sentence shortening for clarity

parent 32ad8ed1
No related branches found
No related tags found
No related merge requests found
......@@ -3,17 +3,17 @@ from Bio import pairwise2
import pickle
import string
summarized_sentences = pickle.load(open('data/in/summarized_sentences.pkl', 'rb'))
shortened_sentences = pickle.load(open('data/in/summarized_sentences.pkl', 'rb'))
with open('data/in/sentence_summarization_vocab.txt', encoding='utf-8') as f:
vocab = [line.strip() for line in f.readlines() if line.strip()]
def shorten(sentence, num_tokens, do_resolve_unks=True):
try:
summarized_sentence = summarized_sentences[sentence.lower()][num_tokens]
shortened_sentence = shortened_sentences[sentence.lower()][num_tokens]
if do_resolve_unks:
summarized_sentence = resolve_unks(sentence, summarized_sentence)
return summarized_sentence
shortened_sentence = resolve_unks(sentence, shortened_sentence)
return shortened_sentence
except KeyError:
print(sentence)
with open('missing-sentences_' + str(num_tokens) + '.txt', 'a', encoding='utf-8') as f:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment