Skip to content
Snippets Groups Projects
Commit 657eb8e5 authored by Victor Zimmermann's avatar Victor Zimmermann
Browse files

Started renaming and commenting code.

parent 755bc6f9
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 242 additions and 134 deletions
src/absinth.py
+ 242
− 134
View file @ 657eb8e5
#!/usr/bin/env python3
import sys
import matplotlib
matplotlib.use("Agg")
print('[A] Loading ' + sys.argv[0] + '.\n')
import os # for reading files
import networkx as nx # for visualisation
from copy import deepcopy
from nltk.corpus import stopwords
import numpy as np # for calculations
import config
import re
import spacy # for nlp
from multiprocessing import Pool
import random
import matplotlib.pyplot as plt
import config
nlp = spacy.load('en') # standard english nlp
#counts occurences of nodes and cooccurrences
def frequencies(corpus_path, target, results):
max_nodes = config.max_nodes
max_edges = config.max_edges
def frequencies(target_string, search_result_list):
"""Counts occurrences of nodes and cooccurrences.
results = [r.replace('<b>', '').replace('</b>', '').replace(r'\\', '').strip() for r in results]
node_freq, edge_freq = process_file(results, target) #initialises frequencies with counts from results
Iterates over the corpus (and snippets provided with the task) line by line
and counts every token and tuple of tokens within a line (context). These
tokens is filtered by stop words, pos tags and context length.
files = [corpus_path + f for f in os.listdir(corpus_path)] #file names of corpus files
i = 0 #for update print statements
for f in files:
Args:
target_string: contexts are selected if they contain this string. For
further processing this string is removed from the contexts.
search_result_list: List of titles and snippets provided with the task.
if i % int(len(files)/11) == 0: #prints update after every 10th of the corpus is parsed
Returns:
node_freq_dict: Dictionary of occurrences of every eligible token
within every context the target occurs in.
edge_freq_dict: Dictionary of occurrences of every eligible tuple of
tokens within every context the target occurs in.
"""
corpus_path = config.corpus
max_node_count = config.max_nodes
max_edge_count = config.max_edges
bracketed_target_string = '('+target_string+')'
# Remove unnecessary tokens from snippets
_search_result_list = list()
for r in search_result_list:
r = r.replace('<b>', '')
r = r.replace('</b>', '')
r = r.replace(r'\\', '')
r = r.strip()
_search_result_list.append(r)
#initialises frequencies with counts from results
node_freq_dict, edge_freq_dict = process_file(_search_result_list,
target_string,
dict(),
dict())
#names of corpus files
corpus_file_path_list = [corpus_path + f for f in os.listdir(corpus_path)]
corpus_size = len(corpus_file_path_list)
processed_file_count = 0
for corpus_file_path in corpus_file_path_list:
node_count = len(node_freq_dict)
edge_count = len(edge_freq_dict)
#prints update after every 11th of the corpus is parsed
if processed_file_count % int(corpus_size/11) == 0:
file_ratio = i/len(files[:])
max_node_ratio = len(node_freq)/max_nodes
max_edge_ratio = len(edge_freq)/max_edges
file_ratio = processed_file_count / corpus_size
max_node_ratio = node_count / max_node_count
max_edge_ratio = edge_count / max_edge_count
ratios = [file_ratio, max_node_ratio, max_edge_ratio]
#uses the ratio closest to 100%.
percentage = int((max(ratios))*100)
highest_ratio = int((max(ratios))*100)
print('[a] ~{:02d}%\tNodes: {}\tEdges: {}.'.format(percentage, len(node_freq), len(edge_freq))+'\t('+target+')')
print('[a] ~{:02d}%\tNodes: {}\tEdges: {}\t{}.'.format(highest_ratio,
node_count,
edge_count,
bracketed_target_string))
#checks maximum node values
if len(node_freq) > max_nodes:
print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq))+'\t('+target+')')
return node_freq, edge_freq
#checks maximum edge values
if len(edge_freq) > max_edges:
print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq))+'\t('+target+')')
return node_freq, edge_freq
with open(f, 'r') as lines: #parses single file
if node_count > max_node_count:
print('[a] 100%\tNodes: {}\tEdges: {}\t{}.'.format(node_count,
edge_count,
bracketed_target_string))
return node_freq_dict, edge_freq_dict
if edge_count > max_edge_count:
print('[a] 100%\tNodes: {}\tEdges: {}\t{}.'.format(node_count,
edge_count,
bracketed_target_string))
return node_freq_dict, edge_freq_dict
with open(corpus_file_path, 'r') as corpus_file:
node_freq, edge_freq = process_file(lines, target, node_freq, edge_freq)
node_freq_dict, edge_freq_dict = process_file(corpus_file,
target_string,
node_freq_dict,
edge_freq_dict)
i += 1
processed_file_count += 1
#update print
print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq))+'\t('+target+')')
print('[a] 100%\tNodes: {}\tEdges: {}\t{}.'.format(node_count,
edge_count,
bracketed_target_string))
return node_freq, edge_freq
return node_freq_dict, edge_freq_dict
def process_file(lines, target, node_freq=None, edge_freq=None):
if node_freq is None:
node_freq = dict()
if edge_freq is None:
edge_freq = dict()
s_target = target.replace('_', ' ') #target word with spaces
stop_words = set(stopwords.words('english') + config.stop_words)
allowed_tags = config.allowed_tags
def process_file(context_list, target_string, node_freq_dict, edge_freq_dict):
"""Updates the counts of nodes and edges for a given document and target.
Ammends the input dictionaries with counts from each context withing the
list of contexts. Furthermore filters out small contexts and tokens from
the stopword list or with wrong pos tags.
Args:
context_list: List of contexts (lines, paragraphs) that are to be
considered for updating the counting dictionaries.
target_string: Target string for filtering out every context that does
not contain it.
node_freq_dict: Dictionary of occurrences of every eligible token
within every context the target occurs in.
edge_freq_dict: Dictionary of occurrences of every eligible tuple of
tokens within every context the target occurs in.
Returns:
node_freq_dict: Updated version of the input node dict.
edge_freq_dict: Updated version of the input edge dict.
"""
spaced_target_string = target_string.replace('_', ' ')
stopword_list = set(stopwords.words('english') + config.stop_words)
allowed_tag_list = config.allowed_tags
min_context_size = config.min_context_size
try:
for line in lines: #parses single paragraph
line = line.lower()
for context in context_list:
if s_target in line: #greedy pre selection, not perfect
context = context.lower()
if spaced_target_string in context: #greedy pre selection, not perfect
tokens = set() #set of node candidates
doc = nlp(line.replace(s_target, target)) #nlp processing
token_set = set() #set of node candidates
if target in [t.text for t in doc]: #better selection
#This replacement allows target to be treated as single entity.
context = context.replace(spaced_target_string, target_string)
processed_context = nlp(context)
if target_string in [token.text for token in processed_context]:
for tok in doc:
text = tok.text #string value
tag = tok.tag_ #pos tag
for token in processed_context:
#doesn't add target word to nodes
if text == target:
if token.text == target_string:
pass
#doesn't add stop words to nodes
elif text in stop_words:
elif token.text in stopword_list:
pass
#only adds tokens with allowed tags to nodes
elif tag in allowed_tags:
tokens.add(tok.text)
elif token.tag_ in allowed_tag_list:
token_set.add(token.text)
#if there are enough (good) tokens in paragraph
if len(tokens) >= min_context_size:
for token in tokens:
context_size = len(token_set)
if context_size >= min_context_size:
for token in token_set:
#updates counts for nodes
if token in node_freq:
node_freq[token] += 1
if token in node_freq_dict:
node_freq_dict[token] += 1
else:
node_freq[token] = 1
node_freq_dict[token] = 1
for edge in {(x,y) for x in tokens for y in tokens if x < y}:
#set of possible edges
for edge in {(x,y) for x in token_set for y in token_set if x < y}:
#updates counts for edges
if edge in edge_freq:
edge_freq[edge] += 1
if edge in edge_freq_dict:
edge_freq_dict[edge] += 1
else:
edge_freq[edge] = 1
edge_freq_dict[edge] = 1
#if a file is corrupted (can't always be catched with if-else)
except UnicodeDecodeError:
pass
#print('Failed to decode:', f)
pass
return node_freq, edge_freq
return node_freq_dict, edge_freq_dict
#build graph from frequency dictionaries
def build_graph(node_freq, edge_freq):
def build_graph(node_freq_dict, edge_freq_dict):
"""Builds undirected weighted graph from dictionaries.
Creates graph and appends every edge and node in the parameter dictionaries,
given they occur frequently enough. For every edge a weight is calculated.
Args:
node_freq_dict: Dictionary of occurrences of every eligible token
within every context the target occurs in.
edge_freq_dict: Dictionary of occurrences of every eligible tuple of
tokens within every context the target occurs in.
Returns:
cooccurence_graph: Filtered undirected dice weighted small word
cooccurence graph for a given target entity.
"""
min_node_freq = config.min_node_freq
min_edge_freq = config.min_edge_freq
max_weight = config.max_weight
G = nx.Graph()
cooccurence_graph = nx.Graph()
#node : node frequency
for key, value in node_freq.items():
for node, frequency in node_freq_dict.items():
if value >= min_node_freq:
G.add_node(key)
if frequency >= min_node_freq:
cooccurence_graph.add_node(node)
#edge : edge frequency
for key, value in edge_freq.items():
for node_tuple, frequency in edge_freq_dict.items():
if frequency < min_edge_freq:
continue
if value < min_edge_freq:
elif node_tuple[0] not in cooccurence_graph.nodes:
continue
if key[0] not in G.nodes or key[1] not in G.nodes:
elif node_tuple[1] not in cooccurence_graph.nodes:
continue
weight = 1 - max(edge_freq[key]/node_freq[key[0]], edge_freq[key]/node_freq[key[1]])
if weight <= max_weight:
G.add_edge(*key, weight=weight)
else:
cooccurrence_frequency = edge_freq_dict[node_tuple]
node0_frequency = node_freq_dict[node_tuple[0]]
node1_frequency = node_freq_dict[node_tuple[1]]
prob_0 = cooccurrence_frequency / node0_frequency
prob_1 = cooccurrence_frequency / node1_frequency
#best_weight = 1 - max(prob_0, prob_1)
dice_weight = 1 - ((prob_0 + prob_1) / 2)
if dice_weight <= max_weight:
cooccurence_graph.add_edge(*node_tuple, weight=dice_weight)
else:
pass
return G
return cooccurence_graph
#Identifies senses by choosing nodes with high degrees
def root_hubs(graph, edge_freq, min_neighbors=4, theshold=0.8):
def root_hubs(graph, edge_freq_dict, min_neighbors=4, theshold=0.8):
min_neighbors = config.min_neighbors
threshold = config.threshold
......@@ -177,7 +281,7 @@ def root_hubs(graph, edge_freq, min_neighbors=4, theshold=0.8):
if G.degree[v] >= min_neighbors:
mfn = sorted(G.adj[v], key=lambda key: edge_freq[v,key] if v < key else edge_freq[key, v], reverse=True)[:min_neighbors] #most frequent neighbors
mfn = sorted(G.adj[v], key=lambda key: edge_freq_dict[v,key] if v < key else edge_freq_dict[key, v], reverse=True)[:min_neighbors] #most frequent neighbors
if np.mean([G.edges[v,n]['weight'] for n in mfn]) < theshold: #if the median weight of the most frequent neighbors is under threshold
......@@ -202,11 +306,11 @@ def root_hubs(graph, edge_freq, min_neighbors=4, theshold=0.8):
#Components algorithm from Véronis (2004), converts graph for target into a MST
def components(graph, hubs, target):
def components(graph, hubs, target_string):
G = deepcopy(graph)
H = hubs #root hubs
t = target
t = target_string
#G.add_node(t)
#for h in H:
......@@ -246,12 +350,12 @@ def score(graph, from_node, to_node):
# Basically Word Sense Disambiguation, matches context to sense
def disambiguate(mst, hubs, contexts, target):
def disambiguate(mst, hubs, contexts, target_string):
target = target.replace('_', ' ')
target_string = target_string.replace('_', ' ')
T = mst #minimum spanning tree
H = hubs #root hubs
C = [c.lower().strip().replace(target, '') for c in contexts] #cleaned up contexts
C = [c.lower().strip().replace(target_string, '') for c in contexts] #cleaned up contexts
score_dict = dict() #memoisation for scores
mapping_dict = {topic:[] for topic in range(1,len(H)+1)} #output of function
......@@ -312,80 +416,73 @@ def disambiguate(mst, hubs, contexts, target):
return mapping_dict
def draw_graph(G, name):
nx.draw_networkx(G,pos=nx.spring_layout(G), with_labels=True, node_size=40, font_size=9, node_color='#2D98DA')
plt.savefig('../figures/'+name+'.png', dpi=200, bbox_inches='tight')
plt.clf()
# our main function, here the main stepps for word sense induction are called
def WSI(topic_id, topic_name, results):
def word_sense_induction(topic_id, topic_name, results):
#buffer for useful information
out_buffer = '\n'
#paths for input (corpus) and output(directory)
corpus_path = config.corpus
output_path = config.output
#path for output(directory)
output_path = './test/'#config.output
#removes trailing new_lines
old_target = topic_name.strip() #original target
old_target_string = topic_name.strip() #original target
if old_target.strip() in [f.replace('.absinth', '') for f in os.listdir(config.output)]:
if old_target_string.strip() in [f.replace('.absinth', '') for f in os.listdir(config.output)]:
return None
out_buffer += ("[A] Word sense induction for '"+old_target+"':\n")
out_buffer += ("[A] Word sense induction for '"+old_target_string+"':\n")
#in topics longer than two words, the leading 'the' can generally be removed without changing the sense
if old_target[:4] == 'the_' and old_target.count('_') >= 2:
if old_target_string[:4] == 'the_' and old_target_string.count('_') >= 2:
target = old_target[4:]
target_string = old_target_string[4:]
else:
target = old_target
target_string = old_target_string
#writes headline for output files
f = open(output_path+target+'.absinth', 'w')
f = open(output_path+target_string+'.absinth', 'w')
f.write('subTopicID\tresultID\n')
#counts occurences of single words, as well as cooccurrences, saves it in dictionary
print('[a]', 'Counting nodes and edges.\t('+old_target+')')
node_freq, edge_freq = frequencies(corpus_path, target, results[topic_id])
print('[a]', 'Counting nodes and edges.\t('+old_target_string+')')
node_freq_dict, edge_freq_dict = frequencies(target_string, results[topic_id])
#builds graph from these dictionaries, also applies multiple filters
print('[a]', 'Building graph.\t('+old_target+')')
G = build_graph(node_freq, edge_freq)
draw_graph(G, topic_name.strip()+'_g')
print('[a]', 'Building graph.\t('+old_target_string+')')
G = build_graph(node_freq_dict, edge_freq_dict)
out_buffer += '[A] Nodes: {}\tEdges: {}\n'.format(str(len(G.nodes)), str(len(G.edges)))
#finds root hubs (senses) within the graph + more filters for these
print('[a]', 'Collecting root hubs.\t('+old_target+')')
H = root_hubs(G, edge_freq)
print('[a]', 'Collecting root hubs.\t('+old_target_string+')')
H = root_hubs(G, edge_freq_dict)
out_buffer += '[A] Root hubs:\n'
#adds sense inventory to buffer with some common neighbors for context
i = 1 #sense index
for h in H:
mfn = sorted(G.adj[h], key=lambda x: edge_freq[h,x] if h < x else edge_freq[x, h], reverse=True)[:6]
mfn = sorted(G.adj[h], key=lambda x: edge_freq_dict[h,x] if h < x else edge_freq_dict[x, h], reverse=True)[:6]
out_buffer += (' {}. {}: {}\n'.format(i, h, ', '.join(mfn)))
i += 1
#performs minimum_spanning_tree algorithm on graph
print('[a]', 'Building minimum spanning tree.\t('+old_target+')')
T = components(G, H, target)
draw_graph(T, topic_name.strip()+'_t')
print('[a]', 'Building minimum spanning tree.\t('+old_target_string+')')
T = components(G, H, target_string)
#matches senses to clusters
print('[a]', 'Disambiguating results.\t('+old_target+')')
D = disambiguate(T, H, results[topic_id], target)
print('[a]', 'Disambiguating results.\t('+old_target_string+')')
D = disambiguate(T, H, results[topic_id], target_string)
out_buffer += ('[A] Mapping: \n')
for cluster,results in D.items():
out_buffer += (' {}. : {}\n'.format(cluster, ', '.join([str(r) for r in results])))
#prints buffer
print('[a]', 'Writing to file.\t('+old_target+')')
print('[a]', 'Writing to file.\t('+old_target_string+')')
print(out_buffer)
#writes clustering to file
......@@ -395,14 +492,8 @@ def WSI(topic_id, topic_name, results):
f.close()
if __name__ == '__main__':
# If absinth.py is run in test environment
if '-t' in sys.argv:
data_path = config.test
else:
data_path = config.dataset
def read_dataset(data_path):
# results.txt includes the queries for a given target word
results = dict()
......@@ -430,10 +521,27 @@ if __name__ == '__main__':
l = line.split('\t')
topics[l[0]] = l[1]
# multiprocessing
with Pool(5) as pool:
# calls WSI() for for topics at a time
pool.starmap(WSI, [(key, value, results) for key,value in topics.items()])
return results, topics
def main():
# If absinth.py is run in test environment
if '-t' in sys.argv:
data_path = config.test
else:
data_path = config.dataset
results, topics = read_dataset(data_path)
with Pool(2) as pool:
parameter_list = [(topic_id, topic_name, results)
for topic_id,topic_name in topics.items()]
pool.starmap(word_sense_induction, parameter_list)
#for key, value in topics.items():
#WSI(key, value, results)
#for topic_id,topic_name in topics.items():
#word_sense_induction(topic_id,topic_name, results)
if __name__ == '__main__':
main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment