Reimplement components() + more comment reform.

    bracketed_target_string = '('+target_string+')'

    # Remove unnecessary tokens from snippets 

    # Remove unnecessary tokens from snippets.

    _search_result_list = list()

    for r in search_result_list:

        r = r.replace('<b>', '')

        r = r.strip()

        _search_result_list.append(r)

    #initialises frequencies with counts from results

    # Initialise frequencies with counts from results.

    node_freq_dict, edge_freq_dict = process_file(_search_result_list,

                                                  target_string,

                                                  dict(),

                                                  dict()) 

    #names of corpus files

    corpus_file_path_list = [corpus_path + f for f in os.listdir(corpus_path)]

    corpus_size = len(corpus_file_path_list)

        node_count = len(node_freq_dict)

        edge_count = len(edge_freq_dict)

        #prints update after every 11th of the corpus is parsed

        # Print update after every 11th of the corpus is parsed.

        if processed_file_count % int(corpus_size/11) == 0: 

            file_ratio = processed_file_count / corpus_size

            ratios = [file_ratio, max_node_ratio, max_edge_ratio]

            #uses the ratio closest to 100%.

            # Use ratio closest to 100%.

            highest_ratio = int((max(ratios))*100)

            print('[a] ~{:02d}%\tNodes: {}\tEdges: {}\t{}.'.format(highest_ratio,

                                                                   edge_count,

                                                                   bracketed_target_string))

        #checks maximum node values

        if node_count > max_node_count:

            print('[a] 100%\tNodes: {}\tEdges: {}\t{}.'.format(node_count,

                                                               edge_count,

        for context in context_list:

            context = context.lower()

            if spaced_target_string in context: #greedy pre selection, not perfect

            if spaced_target_string in context: # Pre-select lines greedy.

                token_set = set() #set of node candidates

                token_set = set()

                #This replacement allows target to be treated as single entity.

                # Allow target to be treated as single entity.

                context = context.replace(spaced_target_string, target_string)

                processed_context = nlp(context)

                    for token in processed_context:

                        #doesn't add target word to nodes

                        # Do not add target word to nodes.

                        if token.text == target_string:

                            pass

                        #doesn't add stop words to nodes

                        # Do not add stop words to nodes.

                        elif token.text in stopword_list:

                            pass

                        #only adds tokens with allowed tags to nodes

                        # Add only tokens with allowed tags to nodes.

                        elif token.tag_ in allowed_tag_list:

                            token_set.add(token.text)

                            else:

                                edge_freq_dict[edge] = 1

    #if a file is corrupted (can't always be catched with if-else)

    # If file is corrupted (can't always be catched with if-else), ignore file.

    except UnicodeDecodeError:

        pass            

    return node_freq_dict, edge_freq_dict

#build graph from frequency dictionaries

def build_graph(node_freq_dict, edge_freq_dict):

    """Builds undirected weighted graph from dictionaries.

    cooccurence_graph = nx.Graph()

    #node : node frequency

    for node, frequency in node_freq_dict.items():

        if frequency >= min_node_freq:

            cooccurence_graph.add_node(node)

    #edge : edge frequency

    for node_tuple, frequency in edge_freq_dict.items():

        if frequency < min_edge_freq:

    return cooccurence_graph

#Identifies senses by choosing nodes with high degrees

def root_hubs(graph, edge_freq_dict, min_neighbors=4, theshold=0.8):

def root_hubs(graph, edge_freq_dict):

    """Identifies senses (root hubs) by choosing nodes with high degrees

    Selects root hubs according to the algorithm in Véronis (2004). Nodes with

    high degree and neighbors with low weights (high cooccurence) are chosen

    until there are no more viable candidates. A root hub candidate is every

    node that is not already a hub and is not a neighbor of one.

    Args:

        graph: Weighted undirected graph.

        edge_freq_dict: Dictionary of weights for every tuple in our graph.

    Returns:

        hub_list: List of root hubs, i.e. strings that are selected using the

            algorithm explained above.

    """

    min_neighbors = config.min_neighbors

    threshold = config.threshold

    G = deepcopy(graph)

    V = sorted(G.nodes, key=lambda key: G.degree[key], reverse=True) # sorts according to degree

    H = list() #output list

    # Allow operations on graph without altering original one.

    graph_copy = deepcopy(graph)

    # Sort according to degree (number of neighbors).

    candidate_list = sorted(graph_copy.nodes,

                            key=lambda node: graph_copy.degree[node],

                            reverse=True)

    while V:

    hub_list = list()

        v = V[0] #best hub candidate

    # While there are still candidates, search for root hubs.

    while candidate_list:

        if G.degree[v] >= min_neighbors:

        candidate = candidate_list[0] #best hub candidate

            mfn = sorted(G.adj[v], key=lambda key: edge_freq_dict[v,key] if v < key else edge_freq_dict[key, v], reverse=True)[:min_neighbors] #most frequent neighbors

        if graph_copy.degree[candidate] >= min_neighbors:

            if np.mean([G.edges[v,n]['weight'] for n in mfn]) < theshold: #if the median weight of the most frequent neighbors is under threshold

            by_frequency = lambda node: edge_freq_dict[candidate,node] \

                                         if candidate < node \

                                         else edge_freq_dict[node,candidate]

                H.append(v)

            most_frequent_neighbor_list = sorted(graph_copy.adj[candidate],

                                                 key=by_frequency,

                                                 reverse=True) [:min_neighbors] 

                #removes neighbors of new hub as hub candidates

                for nbr in deepcopy(G).adj[v]:

            # If the mean weight of the most frequent neighbors cooccur

            # frequently enough with candidate, the candidate is approved.

            if np.mean([graph_copy.edges[candidate,node]['weight'] 

                        for node in most_frequent_neighbor_list])  < threshold: 

                    G.remove_node(nbr)

                # Add candidate as root hub.

                hub_list.append(candidate)

            #removes hub candidate

            G.remove_node(v)

                # Remove neighbors of new hub as hub candidates.

                for neighbor in deepcopy(graph_copy).adj[candidate]:

                    graph_copy.remove_node(neighbor)

            #reorderd potential hubs after deletions

            V = sorted(G.nodes, key=lambda key: G.degree[key], reverse=True)

            # Remove hub candidate.

            graph_copy.remove_node(candidate)

            # Reorder potential hubs after deletions.

            candidate_list = sorted(graph_copy.nodes,

                                    key=lambda node: graph_copy.degree[node],

                                    reverse=True)

        else:

            return H

            return hub_list

    return hub_list

def components(graph, root_hub_list, target_string):

    """Builds minimum spanning tree from graph and removes singletons.

    Applies components algorithm from Véronis (2004) and removes singletons.

    Args:

        graph: Undirected weighted graph.

        root_hub_list: List of strings of root hubs of graph.

        target_string: Root of minimum spanning tree.

    Returns:

        minimum_spanning_tree: Minimum spanning tree with target as

            root and root hubs as direct children. Singletons removed.

    """

    graph_copy = deepcopy(graph)

    return H

    graph_copy.add_node(target_string)

    for root_hub in root_hub_list:

        graph_copy.add_edge(target_string,root_hub,weight=0)

    minimum_spanning_tree = nx.minimum_spanning_tree(graph_copy)

#Components algorithm from Véronis (2004), converts graph for target into a MST

def components(graph, hubs, target_string):

    # Remove singletons, deepcopy for iteration while being altered.

    for node in deepcopy(minimum_spanning_tree).nodes:

        if len(minimum_spanning_tree.adj[node]) == 0:

            minimum_spanning_tree.remove_node(node)

    G = deepcopy(graph)

    H = hubs #root hubs

    t = target_string

    return minimum_spanning_tree

    #G.add_node(t)

    #for h in H:

        #G.add_edge(t,h,weight=0)

    T = nx.minimum_spanning_tree(G)

def score(graph, component, root_hub_list):

    """Calculate score for a given component in a minimum spanning tree.

    #removes singletons

    for node in deepcopy(T).nodes:

        if len(T.adj[node]) == 0:

            T.remove_node(node)

    First the correct root for the component is chosen. If no root hub is

    suitable, an empty array is returned. A score is calculated for the distance

    of the component and its root and returned as part of an array filled with

    zeroes.

    return T

    Args:

        graph: Minimum spanning tree.

        component: Node (string) from which the distances are to be calculated.

        root_hub_list: List of strings of root hubs (senses) of original graph.

    Returns:

        score_array: Array with one score for the correct root hub and filled 

            with zeroes..

    """

    root_hub_count = len(root_hub_list)

    #Initialise score array.

    score_array = np.zeros(root_hub_count)

#Calculates score for a given path in a minimum spanning tree

def score(graph, from_node, to_node):

    # Find root of component.

    distance_list = list()

    for root_hub in root_hub_list:

        if nx.has_path(graph, component, root_hub):

            distance_list.append(1/(1+len(nx.shortest_path(graph, component, root_hub))))

        else:

            distance_list.append(0)

    if sum(distance_list) == 0:

        return score_array

    #if correct tree

    if nx.has_path(graph, from_node, to_node):

    root_idx = np.argmax(distance_list)

    root = root_hub_list[root_idx]

        # calculates shortest path (approximation for path with lowest total weight)

        path = nx.shortest_path(graph, from_node, to_node, 'weight')

    shortest_path = nx.shortest_path(graph, component, root, 'weight')

    total_weight = 0

        #adds weights of every sub-path

        for i in range(1, len(path)):

            sub_from, sub_to = path[i-1], path[i]

    # Add weights of every sub-path.

    for i in range(1, len(shortest_path)):

        sub_from, sub_to = shortest_path[i-1], shortest_path[i]

        total_weight += graph[sub_from][sub_to]['weight']

        #the further the path, the lower the score

        return 1/(1+total_weight)

    score_array = np.zeros(root_hub_count)

    score_array[root_idx] = 1/(1+total_weight)

    else:

    return score_array

        return 0

def disambiguate(minimum_spanning_tree, root_hub_list,

                 context_list, target_string):

    """Matches contexts to senses.

# Basically Word Sense Disambiguation, matches context to sense

def disambiguate(mst, hubs, contexts, target_string):

    Adds up scores for each token in a context string and matches the context

    to the root hub with the highest score.

    Args:

        minimum_spanning_tree: Minimum spanning tree with target as root.

        root_hub_list: List of strings of root hubs (senses).

        context_list: List of sentence strings that are to be clustered.

        target_string: String of target word, also root of MST.

    Returns:

        mapping_dict: Dictionary of root hubs (senses) as keys and context ids

            as values.

    """

    target_string = target_string.replace('_', ' ')

    T = mst #minimum spanning tree

    H = hubs #root hubs

    C = [c.lower().strip().replace(target_string, '') for c in contexts] #cleaned up contexts

    context_list = [context.lower().strip().replace(target_string, '')

                    for context in context_list]

    score_dict = dict() #memoisation for scores

    mapping_dict = {topic:[] for topic in range(1,len(H)+1)} #output of function

    mapping_dict = {topic:[] for topic in range(1,len(root_hub_list)+1)}

    #if no sense is found for a target word, we should assume that there only is one sense

    if len(H) == 0: 

    if len(root_hub_list) == 0: 

        return {0:[i for i in range(1, len(C)+1)]}

        return {0:[i for i in range(1, len(context_list)+1)]}

    idx = 0

    for c in C:

    for context in context_list:

        idx += 1 #index based on position in list

        doc = nlp(c) #parsed context

        texts = [tok.text for tok in doc] #tokens

        scores = np.zeros(len(H)) #initialise with zeros for every sense

        for text in texts:

        processed_context = nlp(context)

        text_list = [token.text for token in processed_context] #tokens

            if text in T.nodes: #if word wasn't filtered out

        score_array = np.zeros(len(root_hub_list)) #initialise with zeros for every sense

                new_scores = list() #scores to be added to total scores

        for text in text_list:

                for h in H: #for each hub

            if text in minimum_spanning_tree.nodes: #if word wasn't filtered out

                    if (text, h) in score_dict: #memoisation

                if text in score_dict: #memoisation

                        new_scores.append(score_dict[(text,h)])

                    new_scores = score_dict[text]

                else:

                        new_score = score(T, text, h)

                        new_scores.append(new_score)

                        score_dict[(text,h)] = new_score #memoisation

                    new_score = score(minimum_spanning_tree, 

                                      text, root_hub_list)

                    score_dict[text] = new_score #memoisation

                scores = scores + np.array(new_scores)

                score_array += new_score

            else:

                pass

        #if the disambiguator could not detect a sense, it should return a singleton, ie. nothing

        if np.max(scores) == 0:

        # If disambiguator does not detect a sense, return singleton.

        if np.max(score_array) == 0:

            pass

        else:

            #applies sense with the highest score to context

            max_score = np.max(scores)

            argmax_score = np.argmax(scores)

            # Apply sense with the highest score to context

            max_score = np.max(score_array)

            argmax_score = np.argmax(score_array)

            #clusters begin at 1

            # Clusters begin at 1

            mapping_dict[argmax_score + 1].append(idx)

    return mapping_dict

# our main function, here the main stepps for word sense induction are called

def word_sense_induction(topic_id, topic_name, results):

def word_sense_induction(topic_id, topic_name, result_list):

    #buffer for useful information

    out_buffer = '\n'

    #path for output(directory)

    output_path = './test/'#config.output

    output_path = config.output

    #removes trailing new_lines

    old_target_string = topic_name.strip() #original target

    #counts occurences of single words, as well as cooccurrences, saves it in dictionary

    print('[a]', 'Counting nodes and edges.\t('+old_target_string+')')

    node_freq_dict, edge_freq_dict = frequencies(target_string, results[topic_id])

    node_freq_dict, edge_freq_dict = frequencies(target_string, result_list[topic_id])

    #builds graph from these dictionaries, also applies multiple filters

    print('[a]', 'Building graph.\t('+old_target_string+')')

    T = components(G, H, target_string)

    #matches senses to clusters

    print('[a]', 'Disambiguating results.\t('+old_target_string+')')

    D = disambiguate(T, H, results[topic_id], target_string)

    print('[a]', 'Disambiguating result_list.\t('+old_target_string+')')

    D = disambiguate(T, H, result_list[topic_id], target_string)

    out_buffer += ('[A] Mapping: \n')

    for cluster,results in D.items():

        out_buffer += (' {}. : {}\n'.format(cluster, ', '.join([str(r) for r in results])))

    for cluster,result_list in D.items():

        out_buffer += (' {}. : {}\n'.format(cluster, ', '.join([str(r) for r in result_list])))

    #prints buffer

    print('[a]', 'Writing to file.\t('+old_target_string+')')

    print(out_buffer)

    #writes clustering to file

    for cluster,results in D.items():

        for result in results:

    for cluster,result_list in D.items():

        for result in result_list:

            f.write(topic_id+'.'+str(cluster)+'\t'+topic_id+'.'+str(result)+'\n')

    f.close()

def main():

    # If absinth.py is run in test environment

    # If absinth.py is run in test environment.

    if '-t' in sys.argv:

        data_path = config.test

    else:

    results, topics = read_dataset(data_path)

    with Pool(2) as pool:

    # Enables manual setting of process count.

    if '-p' in sys.argv:

        process_count = int(sys.argv[sys.argv.index('-p') + 1])

    else:

        process_count = 1

    with Pool(process_count) as pool:

        parameter_list = [(topic_id, topic_name, results)

                          for topic_id,topic_name in topics.items()]

        pool.starmap(word_sense_induction, parameter_list)