Saves graphs now.

reimplementation of Véronis' Hyperlex (2004).

Example:

    The function can be called with the following command.:

    The function can be called with the following command:

        $ python3 absinth.py

"""

##########################

#      Dependencies      #

##########################

import sys

print('[a] Loading ' + sys.argv[0] + '.\n')

import config

import json

import networkx as nx # for visualisation

import numpy as np

import os # for reading files

nlp = spacy.load('en') # standard english nlp

##########################

#      Preprocessing     #

##########################

def read_dataset(data_path: str) -> (dict, dict):

    """Collects topics.txt and results.txt.

    return results, topics

##########################

#        Induction       #

##########################

def induce(topic_name: str, result_list: list) -> (nx.Graph, list, dict):

    """Induces word senses for a given topic from corpus.

    Counts frequencies from corpus and search result list, builds graph from

    these counts (with some filters). Root hubs (senses) are collected from

    this graph.

    Args:

        topic_name: Target string.

        result_list: List of search result (context) strings.

    Returns:

        A cooccurrence graph,

            a list of root hub strings (senses) 

            and dictionary of various statistics.

    """

    stat_dict = dict()

    stat_dict['target'] = topic_name

    #in topics longer than two words, the leading 'the' can generally be removed without changing the sense

    if topic_name[:4] == 'the_' and topic_name.count('_') > 1:

        target_string = topic_name[4:]

    else:

        target_string = topic_name

    print('[a]', 'Counting nodes and edges.\t('+topic_name+')')

    #Check if frequencies were already counted before.

    node_dict_name = topic_name+'_node.json'

    edge_dict_name = topic_name+'_edge.json'

    graph_in_existence = False

    for graph_name in os.listdir(config.graph):

        if topic_name in graph_name:

            graph_in_existence = True

            with open(node_dict_name, 'r') as node_file, open(edge_dict_name, 'r') as edge_file:

                node_freq_dict = json.load(node_file)

                edge_freq_dict = json.load(edge_file)

            continue

    if graph_in_existence == False:

        node_freq_dict, edge_freq_dict = frequencies(target_string, result_list)

        with open(node_dict_name, 'w') as node_file, open(edge_dict_name, 'w') as edge_file:

            node_file.write(json.dumps(node_freq_dict))

            edge_file.write(json.dumps(edge_freq_dict))

    #builds graph from these dictionaries, also applies multiple filters

    print('[a]', 'Building graph.\t('+topic_name+')')

    graph = build_graph(node_freq_dict, edge_freq_dict)

    for string in topic_name.split('_'):

        if string in graph.nodes:

            graph.remove_node(string)

    stat_dict['nodes'] = len(graph.nodes)

    stat_dict['edges'] = len(graph.edges)

    #finds root hubs (senses) within the graph + more filters for these

    print('[a]', 'Collecting root hubs.\t('+topic_name+')')

    root_hub_list = root_hubs(graph, edge_freq_dict)

    #adds sense inventory to buffer with some common neighbors for context

    stat_dict['hubs'] = dict()

    for root_hub in root_hub_list:

        by_frequency = lambda node: edge_freq_dict[root_hub,node] \

                                        if root_hub < node \

                                        else edge_freq_dict[node, root_hub]

        most_frequent_neighbor_list = sorted(graph.adj[root_hub],

                                                key=by_frequency, reverse=True) 

        stat_dict['hubs'][root_hub] = most_frequent_neighbor_list[:6]

    return graph, root_hub_list, stat_dict

def frequencies(target_string: str, search_result_list: list) -> (dict, dict):

    """Counts occurrences of nodes and cooccurrences.

    return hub_list

def components(graph: nx.Graph, root_hub_list: list, target_string: str) -> nx.Graph:

    """Builds minimum spanning tree from graph and removes singletons.

    Applies components algorithm from Véronis (2004) and removes singletons.

    Args:

        graph: Undirected weighted graph.

        root_hub_list: List of strings of root hubs of graph.

        target_string: Root of minimum spanning tree.

    Returns:

        Minimum spanning tree with target as root and root hubs as direct

            children. Singletons removed.

    """

    graph_copy = deepcopy(graph)

    graph_copy.add_node(target_string)

    for root_hub in root_hub_list:

        graph_copy.add_edge(target_string,root_hub,weight=0)

    minimum_spanning_tree = nx.minimum_spanning_tree(graph_copy)

    return minimum_spanning_tree

def score(graph: nx.Graph, component: str, root_hub_list: list) -> np.array:

    """Calculate score for a given component in a minimum spanning tree.

    First the correct root for the component is chosen. If no root hub is

    suitable, an empty array is returned. A score is calculated for the distance

    of the component and its root and returned as part of an array filled with

    zeroes.

    Args:

        graph: Minimum spanning tree.

        component: Node (string) from which the distances are to be calculated.

        root_hub_list: List of strings of root hubs (senses) of original graph.

    Returns:

        Array with one score for the correct root hub and filled with zeroes.

    """

    root_hub_count = len(root_hub_list)

    #Initialise score array.

    score_array = np.zeros(root_hub_count)

    # Find root of component.

    distance_list = list()

    for root_hub in root_hub_list:

        if nx.has_path(graph, component, root_hub):

            distance_list.append(1/(1+len(nx.shortest_path(graph, component, root_hub))))

        else:

            distance_list.append(0)

    if sum(distance_list) == 0:

        return score_array

    root_idx = np.argmax(distance_list)

    root = root_hub_list[root_idx]

    shortest_path = nx.shortest_path(graph, component, root, 'weight')

    total_weight = 0

    # Add weights of every sub-path.

    for i in range(1, len(shortest_path)):

        sub_from, sub_to = shortest_path[i-1], shortest_path[i]

        total_weight += graph[sub_from][sub_to]['weight']

    score_array = np.zeros(root_hub_count)

    score_array[root_idx] = 1/(1+total_weight)

    return score_array

def induce(topic_name: str, result_list: list) -> (nx.Graph, list, dict):

    """Induces word senses for a given topic from corpus.

    Counts frequencies from corpus and search result list, builds graph from

    these counts (with some filters). Root hubs (senses) are collected from

    this graph.

    Args:

        topic_name: Target string.

        result_list: List of search result (context) strings.

    Returns:

        A cooccurrence graph,

            a list of root hub strings (senses) 

            and dictionary of various statistics.

    """

    stat_dict = dict()

    stat_dict['target'] = topic_name

    #in topics longer than two words, the leading 'the' can generally be removed without changing the sense

    if topic_name[:4] == 'the_' and topic_name.count('_') > 1:

        target_string = topic_name[4:]

    else:

        target_string = topic_name

    print('[a]', 'Counting nodes and edges.\t('+topic_name+')')

    node_freq_dict, edge_freq_dict = frequencies(target_string, result_list)

    #builds graph from these dictionaries, also applies multiple filters

    print('[a]', 'Building graph.\t('+topic_name+')')

    graph = build_graph(node_freq_dict, edge_freq_dict)

    for string in topic_name.split('_'):

        if string in graph.nodes:

            graph.remove_node(string)

    stat_dict['nodes'] = len(graph.nodes)

    stat_dict['edges'] = len(graph.edges)

    #finds root hubs (senses) within the graph + more filters for these

    print('[a]', 'Collecting root hubs.\t('+topic_name+')')

    root_hub_list = root_hubs(graph, edge_freq_dict)

##############################

# Propagation Disambiguation #

##############################

    #adds sense inventory to buffer with some common neighbors for context

    stat_dict['hubs'] = dict()

    for root_hub in root_hub_list:

def label_graph(graph: nx.Graph, root_hub_list: list) -> nx.Graph:

    """propagations graph accoring to root hubs.

        by_frequency = lambda node: edge_freq_dict[root_hub,node] \

                                        if root_hub < node \

                                        else edge_freq_dict[node, root_hub]

        most_frequent_neighbor_list = sorted(graph.adj[root_hub],

                                                key=by_frequency, reverse=True) 

        stat_dict['hubs'][root_hub] = most_frequent_neighbor_list[:6]

    return graph, root_hub_list, stat_dict

def colour_graph(graph: nx.Graph, root_hub_list: list) -> nx.Graph:

    """Colours graph accoring to root hubs.

    Evolving network that colours neighboring nodes iterative. See sentiment

    Evolving network that propagations neighboring nodes iterative. See sentiment

    propagation.

    Args:

        root_hub_list: List of senses.

    Returns:

        Coloured graph.

        labelled graph.

    """

            graph.node[node]['sense'] = None

    max_iteration_count = config.max_colour_iteration_count

    max_iteration_count = config.max_propagation_iteration_count

    iteration_count = 0

    stable = False

                graph.node[node]['dist'].append(neighbor_weight_list)

                old_colour = graph_copy.node[node]['sense']

                new_colour = np.argmax(np.mean(graph.node[node]['dist'], axis=0))

                old_propagation = graph_copy.node[node]['sense']

                new_propagation = np.argmax(np.mean(graph.node[node]['dist'], axis=0))

                if old_colour != new_colour:

                if old_propagation != new_propagation:

                    stable = False

                    graph.node[node]['sense'] = new_colour

                    graph.node[node]['sense'] = new_propagation

                else:

                    pass

    return graph

def disambiguate_colour(graph: nx.Graph, root_hub_list: list, context_list: list) -> dict:

    """Clusters senses to root hubs using a coloured graph.

def disambiguate_propagation(graph: nx.Graph, root_hub_list: list, context_list: list) -> dict:

    """Clusters senses to root hubs using a labelled graph.

    This algorithm colours the graph using evolutionary graph theory

    This algorithm propagations the graph using evolutionary graph theory

    and calculates scores for each root hub given a context based on this graph.

    Args:

        A dictionary with root hub IDs as keys and context indices as values.

    """

    coloured_graph = colour_graph(graph, root_hub_list)

    labelled_graph = label_graph(graph, root_hub_list)

    mapping_dict = {i:list() for i in range(1,len(root_hub_list)+1)}

            else:

                text = token.text

            if text in coloured_graph.nodes:

            if text in labelled_graph.nodes:

                text_colour_dist = coloured_graph.node[text]['dist']

                text_propagation_dist = labelled_graph.node[text]['dist']

                if not any(text_colour_dist):

                if not any(text_propagation_dist):

                    pass

                        root_hub_idx = root_hub_list.index(root_hub)

                        if nx.has_path(coloured_graph , text, root_hub):

                        if nx.has_path(labelled_graph , text, root_hub):

                            shortest_path = nx.shortest_path(coloured_graph ,

                            shortest_path = nx.shortest_path(labelled_graph ,

                                                            text,

                                                            root_hub,

                                                            'weight')

                            for i in range(1, len(shortest_path)):

                                sub_from, sub_to = shortest_path[i-1], shortest_path[i]

                                total_weight += \

                                    coloured_graph[sub_from][sub_to]['weight']

                                    labelled_graph[sub_from][sub_to]['weight']

                            score[root_hub_idx] += (1/(1+total_weight)) \

                             * coloured_graph.node[text]['dist'][root_hub_idx]

                             * labelled_graph.node[text]['dist'][root_hub_idx]

                        else:

    return mapping_dict

##############################

#     MST Disambiguation     #

##############################

def components(graph: nx.Graph, root_hub_list: list, target_string: str) -> nx.Graph:

    """Builds minimum spanning tree from graph and removes singletons.

    Applies components algorithm from Véronis (2004) and removes singletons.

    Args:

        graph: Undirected weighted graph.

        root_hub_list: List of strings of root hubs of graph.

        target_string: Root of minimum spanning tree.

    Returns:

        Minimum spanning tree with target as root and root hubs as direct

            children. Singletons removed.

    """

    graph_copy = deepcopy(graph)

    graph_copy.add_node(target_string)

    for root_hub in root_hub_list:

        graph_copy.add_edge(target_string,root_hub,weight=0)

    minimum_spanning_tree = nx.minimum_spanning_tree(graph_copy)

    return minimum_spanning_tree

def score(graph: nx.Graph, component: str, root_hub_list: list) -> np.array:

    """Calculate score for a given component in a minimum spanning tree.

    First the correct root for the component is chosen. If no root hub is

    suitable, an empty array is returned. A score is calculated for the distance

    of the component and its root and returned as part of an array filled with

    zeroes.

    Args:

        graph: Minimum spanning tree.

        component: Node (string) from which the distances are to be calculated.

        root_hub_list: List of strings of root hubs (senses) of original graph.

    Returns:

        Array with one score for the correct root hub and filled with zeroes.

    """

    root_hub_count = len(root_hub_list)

    #Initialise score array.

    score_array = np.zeros(root_hub_count)

    # Find root of component.

    distance_list = list()

    for root_hub in root_hub_list:

        if nx.has_path(graph, component, root_hub):

            distance_list.append(1/(1+len(nx.shortest_path(graph, component, root_hub))))

        else:

            distance_list.append(0)

    if sum(distance_list) == 0:

        return score_array

    root_idx = np.argmax(distance_list)

    root = root_hub_list[root_idx]

    shortest_path = nx.shortest_path(graph, component, root, 'weight')

    total_weight = 0

    # Add weights of every sub-path.

    for i in range(1, len(shortest_path)):

        sub_from, sub_to = shortest_path[i-1], shortest_path[i]

        total_weight += graph[sub_from][sub_to]['weight']

    score_array = np.zeros(root_hub_count)

    score_array[root_idx] = 1/(1+total_weight)

    return score_array

def disambiguate_mst(graph: nx.Graph, root_hub_list: list,

                 context_list: list, topic_name: str) -> dict:

    """Matches contexts to senses.

    return mapping_dict

def print_stats(stat_dict: dict) -> None:

    """Prints various statistics and logs them to file.

    Args:

        stat_dict: Dictionary with various statistics.

    """

    stat_string = []

    ts = time.gmtime()

    key_list= ['target','nodes','edges','L','C','L_rand','C_rand','clusters','a_mean_size','h_mean_size','pipe_gain']

    stat_string.append('Topic: {}.'.format(stat_dict['target']))

    stat_string.append('Processed {} at {}.'.format(time.strftime("%Y-%m-%d", ts),time.strftime("%H:%M:%S", ts)))

    stat_string.append('Nodes: {}\tEdges: {}.'.format(stat_dict['nodes'],stat_dict['edges']))

    stat_string.append('Characteristic path length: {}.'.format(stat_dict['L']))

    stat_string.append('Global clustering coefficient: {}.'.format(stat_dict['C']))

    stat_string.append('Mean cluster length (arithmetic): {}.'.format(stat_dict['a_mean_size']))

    stat_string.append('Mean cluster length (harmonic): {}.'.format(stat_dict['h_mean_size']))

    stat_string.append('Number of clusters: {}.'.format(stat_dict['clusters']))

    stat_string.append('Tuples gained through merging: {}.'.format(stat_dict['pipe_gain']))

    stat_string.append('Sense inventory:')

    for hub in stat_dict['hubs'].keys():

        stat_string.append(' -> {}: {}.'.format(hub, ", ".join(stat_dict['hubs'][hub])))

    print('\n[A] '+'\n[A] '.join(stat_string)+'\n')

    with open('statistics.txt', 'a') as stat_file:

        stat_file.write('\n '.join(stat_string)+'\n\n')

    write_header = not os.path.exists('.statistics.tsv')

    with open('.statistics.tsv', 'a') as stat_file:

        if write_header:

            stat_file.write('\t'.join(key_list)+'\n')

        stat_file.write('\t'.join([str(stat_dict[key]) for key in key_list])+'\n')

##############################

#          Statistics        #

##############################

def global_clustering_coefficient(graph: nx.Graph) -> float:

    return np.mean(path_length_list)

def print_stats(stat_dict: dict) -> None:

    """Prints various statistics and logs them to file.

    Args:

        stat_dict: Dictionary with various statistics.

    """

    stat_string = []

    ts = time.gmtime()

    key_list= ['target','nodes','edges','L','C','L_rand','C_rand','clusters','a_mean_size','h_mean_size','pipe_gain']

    stat_string.append('Topic: {}.'.format(stat_dict['target']))

    stat_string.append('Processed {} at {}.'.format(time.strftime("%Y-%m-%d", ts),time.strftime("%H:%M:%S", ts)))

    stat_string.append('Nodes: {}\tEdges: {}.'.format(stat_dict['nodes'],stat_dict['edges']))

    stat_string.append('Characteristic path length: {}.'.format(stat_dict['L']))

    stat_string.append('Global clustering coefficient: {}.'.format(stat_dict['C']))

    stat_string.append('Mean cluster length (arithmetic): {}.'.format(stat_dict['a_mean_size']))

    stat_string.append('Mean cluster length (harmonic): {}.'.format(stat_dict['h_mean_size']))

    stat_string.append('Number of clusters: {}.'.format(stat_dict['clusters']))

    stat_string.append('Tuples gained through merging: {}.'.format(stat_dict['pipe_gain']))

    stat_string.append('Sense inventory:')

    for hub in stat_dict['hubs'].keys():

        stat_string.append(' -> {}: {}.'.format(hub, ", ".join(stat_dict['hubs'][hub])))

    print('\n[A] '+'\n[A] '.join(stat_string)+'\n')

    with open('statistics.txt', 'a') as stat_file:

        stat_file.write('\n '.join(stat_string)+'\n\n')

    write_header = not os.path.exists('.statistics.tsv')

    with open('.statistics.tsv', 'a') as stat_file:

        if write_header:

            stat_file.write('\t'.join(key_list)+'\n')

        stat_file.write('\t'.join([str(stat_dict[key]) for key in key_list])+'\n')