Add comments, fixed some bugs, other minor updates.

nlp = spacy.load('en') # standard english nlp

#counts occurences of nodes and cooccurrences

def frequencies(corpus_path, target):

    stop_words = set(stopwords.words('english') + config.stop_words)

    max_nodes = config.max_nodes

    max_edges = config.max_edges

    node_freq = dict()

    edge_freq = dict()

    node_freq = dict() #counts (potential) nodes

    edge_freq = dict() #counts (potential) edges

    files = [corpus_path + f for f in os.listdir(corpus_path)]

    files = [corpus_path + f for f in os.listdir(corpus_path)] #file names of corpus files

    s_target = target.replace('_', ' ') #target word with spaces

    i = 0

    i = 0 #for update print statements

    for f in files:

        if i % int(len(files)/10) == 0:

        if i % int(len(files)/10) == 0: #prints update after every 10th of the corpus is parsed

            file_ratio = i/len(files[:])

            max_node_ratio = len(node_freq)/max_nodes

            ratios = [file_ratio, max_node_ratio, max_edge_ratio]

            #uses the ratio closest to 100%.

            percentage = int((max(ratios))*100)

            print('[a] ~{:02d}%\tNodes: {}\tEdges: {}.'.format(percentage, len(node_freq), len(edge_freq)), target)

        #checks maximum node values

        if len(node_freq) > max_nodes:

            return node_freq, edge_freq

        #checks maximum edge values

        if len(edge_freq) > max_edges:

            return node_freq, edge_freq

        with open(f, 'r') as lines:

        with open(f, 'r') as lines: #parses single file

            try:

                for line in lines:

                for line in lines: #parses single paragraph

                    line = line.lower()

                    if s_target in line:

                    if s_target in line: #greedy pre selection, not perfect

                        tokens = set()

                        doc = nlp(line.replace(s_target, target))

                        tokens = set() #set of node candidates

                        doc = nlp(line.replace(s_target, target)) #nlp processing

                        if target in [t.text for t in doc]:

                        if target in [t.text for t in doc]: #better selection

                            for tok in doc:

                                text = tok.text

                                tag = tok.tag_

                                text = tok.text #string value

                                tag = tok.tag_ #pos tag

                                #doesn't add target word to nodes

                                if text == target:

                                    pass

                                #doesn't add stop words to nodes

                                elif text in stop_words:

                                    pass

                                #only adds tokens with allowed tags to nodes

                                elif tag in allowed_tags:

                                    tokens.add(tok.text)

                            #if there are enough (good) tokens in paragraph

                            if len(tokens) >= min_context_size:

                                for token in tokens:

                                    #updates counts for nodes

                                    if token in node_freq:

                                        node_freq[token] += 1

                                    else:

                                for edge in {(x,y) for x in tokens for y in tokens if x < y}:

                                    #updates counts for edges

                                    if edge in edge_freq:

                                        edge_freq[edge] += 1

                                    else:

                                        edge_freq[edge] = 1

            #if a file is corrupted (can't always be catched with if-else)

            except UnicodeDecodeError:

                pass

        i += 1

    #update print

    print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq)), target)

    return node_freq, edge_freq

#build graph from frequency dictionaries

def build_graph(node_freq, edge_freq):

    min_node_freq = config.min_node_freq

    G = nx.Graph()

    #node : node frequency

    for key, value in node_freq.items():

        if value >= min_node_freq:

            G.add_node(key)

    #edge : edge frequency

    for key, value in edge_freq.items():

        if value < min_edge_freq:

    return G

#Identifies senses by choosing nodes with high degrees

def root_hubs(graph, edge_freq, min_neighbors=4, theshold=0.8):

    min_neighbors = config.min_neighbors

    threshold = config.threshold

    G = deepcopy(graph)

    V = sorted(G.nodes, key=lambda key: G.degree[key], reverse=True) # -1 to sort descending (...3 -> 2 -> 1...)

    H = list()

    V = sorted(G.nodes, key=lambda key: G.degree[key], reverse=True) # sorts according to degree

    H = list() #output list

    while V:

        v = V[0]

        v = V[0] #best hub candidate

        if G.degree[v] >= min_neighbors:

            mfn = sorted(G.adj[v], key=lambda key: edge_freq[v,key] if v < key else edge_freq[key, v], reverse=True)[:min_neighbors] #mfn: most frequent neighbors

            mfn = sorted(G.adj[v], key=lambda key: edge_freq[v,key] if v < key else edge_freq[key, v], reverse=True)[:min_neighbors] #most frequent neighbors

            if np.mean([G.edges[v,n]['weight'] for n in mfn]) < theshold:

            if np.mean([G.edges[v,n]['weight'] for n in mfn]) < theshold: #if the median weight of the most frequent neighbors is under threshold

                H.append(v)

                #removes neighbors of new hub as hub candidates

                for nbr in deepcopy(G).adj[v]:

                    G.remove_node(nbr)

            #removes hub candidate

            G.remove_node(v)

            #reorderd potential hubs after deletions

            V = sorted(G.nodes, key=lambda key: G.degree[key], reverse=True)

        else:

def components(graph, hubs, target):

    G = deepcopy(graph)

    H = hubs

    H = hubs #root hubs

    t = target

    #G.add_node(t)

    T = nx.minimum_spanning_tree(G)

    #removes singletons

    for node in deepcopy(T).nodes:

        if len(T.adj[node]) == 0:

            T.remove_node(node)

    return T

#Calculates score for a given path in a minimum spanning tree

def score(graph, from_node, to_node):

    #if correct tree

    if nx.has_path(graph, from_node, to_node):

        # calculates shortest path (approximation for path with lowest total weight)

        path = nx.shortest_path(graph, from_node, to_node, 'weight')

        total_weight = 0

        #adds weights of every sub-path

        for i in range(1, len(path)):

            sub_from, sub_to = path[i-1], path[i]

            total_weight += graph[sub_from][sub_to]['weight']

        #the further the path, the lower the score

        return 1/(1+total_weight)

    else:

        return 0

# Basically Word Sense Disambiguation, matches context to sense

def disambiguate(mst, hubs, contexts, target=""):

    target = target.replace('_', ' ')

    T = mst

    H = hubs

    C = [c.lower().strip().replace(target, '') for c in contexts]

    T = mst #minimum spanning tree

    H = hubs #root hubs

    C = [c.lower().strip().replace(target, '') for c in contexts] #cleaned up contexts

    score_dict = dict()

    result = list()

    score_dict = dict() #memoisation for scores

    result = list() #output of function

    for c in C:

        idx = C.index(c) + 1

        idx = C.index(c) + 1 #index based on position in list

        #if no sense is found for a target word, we should assume that there only is one sense

        if len(H) == 0: 

            result.append((1, idx))

            result.append((1, idx, 0))

        else:

            doc = nlp(c)

            texts = [tok.text for tok in doc]

            doc = nlp(c) #parsed context

            texts = [tok.text for tok in doc] #tokens

            scores = np.zeros(len(H)) #initialise with zeros for every sense

            for text in texts:

                if text in T.nodes:

                if text in T.nodes: #if word wasn't filtered out

                    new_scores = list()

                    new_scores = list() #scores to be added to total scores

                    for h in H: #for each hub

                        if (text, h) in score_dict: #memoisation

                    for h in H:

                        if (text, h) in score_dict:

                            new_scores.append(score_dict[(text,h)])

                        else:

                            new_score = score(T, text, h)

                            new_scores.append(new_score)

                            score_dict[(text,h)] = new_scores

                            score_dict[(text,h)] = new_score #memoisation

                    scores = np.add(scores, new_scores)

                    scores = scores + np.array(new_scores)

                else:

            else:

                result.append((np.argmax(scores)+1, idx))

                #applies sense with the highest score to context

                max_score = np.max(scores)

                argmax_score = np.argmax(scores)

                #clusters begin at 1

                result.append((argmax_score + 1, idx))

    return result

# our main function, here the main stepps for word sense induction are called

def WSI(topic_id, topic_name, results):

    #buffer for useful information

    out_buffer = '\n'

    #paths for input (corpus) and output(directory)

    corpus_path = config.corpus

    output_path = config.output

    #removes trailing new_lines

    old_target = topic_name.strip() #original target

    out_buffer += ("[A] Word sense induction for '"+old_target+"':\n")

    if old_target[:4] == 'the_' and old_target.count('_') >= 2: #hard coded 'the'-protection

    #in topics longer than two words, the leading 'the' can generally be removed without changing the sense

    if old_target[:4] == 'the_' and old_target.count('_') >= 2:

        target = old_target[4:]

        target = old_target

    #writes headline for output files

    f = open(output_path+target+'.absinth', 'w')

    f.write('subTopicID\tresultID\n')

    #counts occurences of single words, as well as cooccurrences, saves it in dictionary

    print('[a]', 'Counting nodes and edges.', old_target)

    node_freq, edge_freq = frequencies(corpus_path, target)

    out_buffer += '[A] Nodes: {}\tEdges: {}\n'.format(str(len(node_freq)), str(len(edge_freq)))

    #builds graph from these dictionaries, also applies multiple filters

    print('[a]', 'Building graph.', old_target)

    G = build_graph(node_freq, edge_freq)

    #finds root hubs (senses) within the graph + more filters for these

    print('[a]', 'Collecting root hubs.', old_target)

    H = root_hubs(G, edge_freq)

    out_buffer += '[A] Root hubs:\n'

    i = 1

    #adds sense inventory to buffer with some common neighbors for context

    i = 1 #sense index

    for h in H:

        mfn = sorted(G.adj[h], key=lambda x: edge_freq[h,x] if h < x else edge_freq[x, h], reverse=True)[:6]

        out_buffer += (' {}. {}: {}\n'.format(i, h, mfn))

        i += 1

    #performs minimum_spanning_tree algorithm on graph

    print('[a]', 'Building minimum spanning tree.', old_target)

    T = components(G, H, target)

    #matches senses to clusters

    print('[a]', 'Disambiguating results.', old_target)

    D = disambiguate(T, H, results[topic_id], target)

    out_buffer += ('[A] Mapping: '+ str(D) + '\n')

    #prints buffer

    print('[a]', 'Writing to file.', old_target)

    print(out_buffer)

    #writes clustering to file

    for d in D:

        f.write(topic_id+'.'+str(d[0])+'\t'+topic_id+'.'+str(d[1])+'\n')

if __name__ == '__main__':

    # If absinth.py is run in test environment

    if '-t' in sys.argv:

        data_path = config.test

    else:

        data_path = config.dataset

    # results.txt includes the queries for a given target word

    results = dict()

    with open(data_path+'results.txt', 'r') as results_file:

        for line in results_file.readlines()[1:]:

            l = line.split('\t')

            id1, _ = l[0].split('.')

            id1, _ = l[0].split('.') #the second part of the id is ignored, as it is identical to the list index

            if id1 not in results:

                results[id1]=list()

            results[id1].append(" ".join(l[2:]))

            results[id1].append(" ".join(l[2:])) # here I join title and snippet, the URL is ignored

    # topics.txt is a list of target words

    topics = dict()

    with open(data_path+'topics.txt', 'r') as topics_file:

            l = line.split('\t')

            topics[l[0]] = l[1]

    # multiprocessing

    with Pool(4) as pool:

        # calls WSI() for for topics at a time

        pool.starmap(WSI, [(key, value, results) for key,value in topics.items()])

    #for key, value in topics.items():

    #    WSI(key, value, results)