New baseline, naive approach to graph based induction.

#!/usr/bin/env python3

# -*- coding: utf-8 -*-

"""Artificially Basic System Trying to Induce Numerous ENTities

This module performs word sense induction for a given word on a corpus and

matches a list of contexts to each. The method to achieve this is just a 

baseline, working with a random selection of senses and a simple Lesk

implementation.

Example:

    The function can be called with the following command:

        $ python3 abstinent.py

    The function can be called with a list of modifiers.

Modifiers:

    '-t': Runs abstinent.py on the trial path given in the config.py instead of

        the data_path.

    '-p n': Runs abstinent.py with n concurrent processes (standard: 1). 

.. _Association Based Semantic Induction Tools from Heidelberg

    https://gitlab.cl.uni-heidelberg.de/zimmermann/absinth

"""

import sys

print('[a] Loading ' + sys.argv[0] + '.\n')

import config

import networkx as nx # for visualisation

import numpy as np

import os # for reading files

import pprint

import random

import re

import scipy.special

import spacy # for nlp

import time

from copy import deepcopy

from multiprocessing import Pool

from scipy import stats 

random.seed(325)

nlp = spacy.load('en') # standard english nlp

def read_dataset(data_path: str) -> (dict, dict):

    """Collects topics.txt and results.txt.

    Iterates over topics.txt and results.txt in the data path and converts them

    to dictionaries with the ID as key and the target word / title + snippet as

    values.

    Args:

        data_path: File path to directory containing topics.txt and results.txt.

    Returns:

        One dictionary for each file. 

    """

    results = dict()

    with open(data_path+'results.txt', 'r') as results_file:

        for line in results_file.readlines()[1:]:

            l = line.split('\t')

            id1, _ = l[0].split('.') #the second part of the id is ignored, as it is identical to the list index

            if id1 not in results:

                results[id1]=list()

            results[id1].append(" ".join(l[2:]).strip()) # here I join title and snippet, the URL is ignored

    # topics.txt is a list of target words

    topics = dict()

    with open(data_path+'topics.txt', 'r') as topics_file:

        for line in topics_file.readlines()[1:]:

            l = line.split('\t')

            topics[l[0]] = l[1].strip()

    return results, topics

def frequencies(target_string: str, search_result_list: list) -> (dict, dict):

    """Counts occurrences of nodes and cooccurrences.

    Iterates over the corpus (and snippets provided with the task) line by line 

    and counts every token and tuple of tokens within a line (context). These 

    tokens is filtered by stop words, pos tags and context length.

    Args:

        target_string: contexts are selected if they contain this string. For

            further processing this string is removed from the contexts.

        search_result_list: List of titles and snippets provided with the task.

    Returns:

        Dictionary of occurrences of every eligible token within every context

            the target occurs in, dictionary of occurrences of every eligible

            tuple of tokens within every context the target occurs in.

    """

    corpus_path = config.corpus

    max_node_count = config.max_nodes

    max_edge_count = config.max_edges

    bracketed_target_string = '('+target_string+')'

    # Remove unnecessary tokens from snippets.

    _search_result_list = list()

    for r in search_result_list:

        r = r.replace('<b>', '')

        r = r.replace('</b>', '')

        r = r.replace(r'\\', '')

        r = r.strip()

        _search_result_list.append(r)

    # Initialise frequencies with counts from results.

    node_freq_dict, edge_freq_dict = process_file(_search_result_list,

                                                  target_string,

                                                  dict(),

                                                  dict()) 

    corpus_file_path_list = [corpus_path + f for f in os.listdir(corpus_path)]

    corpus_size = len(corpus_file_path_list)

    processed_file_count = 0

    for corpus_file_path in corpus_file_path_list:

        node_count = len(node_freq_dict)

        edge_count = len(edge_freq_dict)

        # Print update after every 11th of the corpus is parsed.

        if processed_file_count % int(corpus_size/11) == 0: 

            file_ratio = processed_file_count / corpus_size

            max_node_ratio = node_count / max_node_count

            max_edge_ratio = edge_count / max_edge_count

            ratios = [file_ratio, max_node_ratio, max_edge_ratio]

            # Use ratio closest to 100%.

            highest_ratio = int((max(ratios))*100)

            print('[a] ~{:02d}%\tNodes: {}\tEdges: {}\t{}.'.format(highest_ratio,

                                                                   node_count,

                                                                   edge_count,

                                                                   bracketed_target_string))

        if node_count > max_node_count:

            print('[a] 100%\tNodes: {}\tEdges: {}\t{}.'.format(node_count,

                                                               edge_count,

                                                               bracketed_target_string))

            return node_freq_dict, edge_freq_dict

        if edge_count > max_edge_count:

            print('[a] 100%\tNodes: {}\tEdges: {}\t{}.'.format(node_count,

                                                               edge_count,

                                                               bracketed_target_string))

            return node_freq_dict, edge_freq_dict

        with open(corpus_file_path, 'r') as corpus_file:

            node_freq_dict, edge_freq_dict = process_file(corpus_file,

                                                          target_string,

                                                          node_freq_dict,

                                                          edge_freq_dict)

        processed_file_count += 1

    print('[a] 100%\tNodes: {}\tEdges: {}\t{}.'.format(node_count,

                                                       edge_count,

                                                       bracketed_target_string))

    return node_freq_dict, edge_freq_dict

def process_file(context_list: list, target_string: str,

                 node_freq_dict: dict, edge_freq_dict: dict) -> (dict, dict): 

    """Updates the counts of nodes and edges for a given document and target.

    Ammends the input dictionaries with counts from each context withing the

    list of contexts. Furthermore filters out small contexts and tokens from

    the stopword list or with wrong pos tags.

    Args:

        context_list: List of contexts (lines, paragraphs) that are to be

            considered for updating the counting dictionaries.

        target_string: Target string for filtering out every context that does 

            not contain it.

        node_freq_dict: Dictionary of occurrences of every eligible token

            within every context the target occurs in.

        edge_freq_dict: Dictionary of occurrences of every eligible tuple of

            tokens within every context the target occurs in.

    Returns:

        Updated versions of the input node dict and input edge dict.

    """

    spaced_target_string = target_string.replace('_', ' ')

    stopword_list = config.stop_words

    allowed_tag_list = config.allowed_tags

    min_context_size = config.min_context_size

    max_context_size = config.max_context_size

    try:

        for context in context_list:

            context = context.lower()

            if spaced_target_string in context: # Pre-select lines greedy.

                token_set = set()

                # Allow target to be treated as single entity.

                context = context.replace(spaced_target_string, target_string)

                processed_context = nlp(context)

                if target_string in [token.text for token in processed_context]:

                    for token in processed_context:

                        # Do not add target word to nodes.

                        if token.text == target_string:

                            pass

                        # Do not add stop words to nodes.

                        elif token.is_stop or token.text in stopword_list:

                            pass

                        # Add only tokens with allowed tags to nodes.

                        elif token.tag_ in allowed_tag_list:

                            if config.lemma == True:

                                token_set.add(token.lemma_)

                            else:

                                token_set.add(token.text)

                    context_size = len(token_set)

                    if context_size >= min_context_size and context_size <= max_context_size:

                        for token in token_set:

                            if token in node_freq_dict:

                                node_freq_dict[token] += 1

                            else:

                                node_freq_dict[token] = 1

                        #set of possible edges

                        for edge in {(x,y) for x in token_set for y in token_set if x < y}:

                            if edge in edge_freq_dict:

                                edge_freq_dict[edge] += 1

                            else:

                                edge_freq_dict[edge] = 1

    # If file is corrupted (can't always be catched with if-else), ignore file.

    except UnicodeDecodeError:

        pass            

    return node_freq_dict, edge_freq_dict

def build_graph(node_freq_dict: dict, edge_freq_dict: dict) -> nx.Graph:

    """Builds undirected weighted graph from dictionaries.

    Creates graph and appends every edge and node in the parameter dictionaries,

    given they occur frequently enough. For every edge a weight is calculated.

    Args:

        node_freq_dict: Dictionary of occurrences of every eligible token

            within every context the target occurs in.

        edge_freq_dict: Dictionary of occurrences of every eligible tuple of

            tokens within every context the target occurs in.

    Returns:

        Filtered undirected dice weighted small word cooccurrence graph for a 

            given target entity.

    """

    min_node_freq = config.min_node_freq

    min_edge_freq = config.min_edge_freq

    max_weight = config.max_weight

    cooccurrence_graph = nx.Graph()

    for node, frequency in node_freq_dict.items():

        if frequency >= min_node_freq:

            cooccurrence_graph.add_node(node)

    for node_tuple, frequency in edge_freq_dict.items():

        if frequency < min_edge_freq:

            continue

        elif node_tuple[0] not in cooccurrence_graph.nodes:

            continue

        elif node_tuple[1] not in cooccurrence_graph.nodes:

            continue

        else:

            cooccurrence_frequency = edge_freq_dict[node_tuple]

            node0_frequency = node_freq_dict[node_tuple[0]]

            node1_frequency = node_freq_dict[node_tuple[1]]

            prob_0 = cooccurrence_frequency / node0_frequency

            prob_1 = cooccurrence_frequency / node1_frequency

            best_weight = 1 - max(prob_0, prob_1)

            #dice_weight = 1 - ((prob_0 + prob_1) / 2)

            if best_weight <= max_weight:

                cooccurrence_graph.add_edge(*node_tuple, weight=best_weight)

            else:

                pass

    # Remove singletons, deepcopy for iteration while being altered.

    for node in deepcopy(cooccurrence_graph).nodes:

        if len(cooccurrence_graph.adj[node]) == 0:

            cooccurrence_graph.remove_node(node)

    return cooccurrence_graph

def induce(topic_name: str, result_list: list) -> (nx.Graph, list, dict):

    """

    Use n random nodes as root hubs.

    """

    stat_dict = dict()

    stat_dict['target'] = topic_name

    print('[a]', 'Counting nodes and edges.\t('+topic_name+')')

    node_freq_dict, edge_freq_dict = frequencies(topic_name, result_list)

    #builds graph from these dictionaries, also applies multiple filters

    print('[a]', 'Building graph.\t('+topic_name+')')

    graph = build_graph(node_freq_dict, edge_freq_dict)

    for string in topic_name.split('_'):

        if string in graph.nodes:

            graph.remove_node(string)

    stat_dict['nodes'] = len(graph.nodes)

    stat_dict['edges'] = len(graph.edges)

    #finds root hubs (senses) within the graph + more filters for these

    print('[a]', 'Collecting root hubs.\t('+topic_name+')')

    sense_count = min(config.sense_count, len(graph.nodes))

    root_hub_list = sorted([(value,key) for key,value in node_freq_dict.items() if key in graph.nodes],

                           reverse=True)[:sense_count]

    root_hub_list = [hub[1] for hub in root_hub_list]

    #adds sense inventory to buffer with some common neighbors for context

    stat_dict['hubs'] = dict()

    for root_hub in root_hub_list:

        by_frequency = lambda node: edge_freq_dict[root_hub,node] \

                                        if root_hub < node \

                                        else edge_freq_dict[node, root_hub]

        most_frequent_neighbor_list = sorted(graph.adj[root_hub],

                                                key=by_frequency, reverse=True) 

        stat_dict['hubs'][root_hub] = most_frequent_neighbor_list[:6]

    return graph, root_hub_list, stat_dict

def bag_of_senses(graph: nx.Graph, root_hub_list:list) -> dict:

    """

    Matches each node to the root hub it is closest to.

    """

    root_hub_count = len(root_hub_list)

    bag = {i:[] for i in range(root_hub_count)}

    for node in graph.nodes:

        score = [0] * root_hub_count

        for i in range(root_hub_count):

            root = root_hub_list[i]

            if nx.has_path(graph, node, root):

                path = nx.shortest_path(graph, node, root, 'weight')

                score[i] = 1/(1+len(path))

        bag[np.argmax(score)].append(node)

    return bag

def disambiguate(bag_of_senses: dict, context_list: list) -> dict:

    """

    Lesk.

    """

    context_idx = 0

    mapping_dict = dict()

    for context in context_list:

        context_idx += 1

        score = [0] * len(bag_of_senses)

        processed_context = nlp(context)

        text_list = [token.text for token in processed_context]

        for text in text_list:

            for sense, words in bag_of_senses.items():

                if text in words:

                    score[sense] += 1

        sense = np.argmax(score)

        if sense in mapping_dict:

            mapping_dict[sense].append(context_idx)

        else:

            mapping_dict[sense] = [context_idx]

    return mapping_dict

def print_stats(stat_dict: dict) -> None:

    """Prints various statistics and logs them to file.

    Args:

        stat_dict: Dictionary with various statistics.

    """

    stat_string = []

    ts = time.gmtime()

    key_list= ['target','nodes','edges','L','C','L_rand','C_rand','clusters','a_mean_size','h_mean_size','pipe_gain']

    stat_string.append('Topic: {}.'.format(stat_dict['target']))

    stat_string.append('Processed {} at {}.'.format(time.strftime("%Y-%m-%d", ts),time.strftime("%H:%M:%S", ts)))

    stat_string.append('Nodes: {}\tEdges: {}.'.format(stat_dict['nodes'],stat_dict['edges']))

    stat_string.append('Characteristic path length: {}.'.format(stat_dict['L']))

    stat_string.append('Global clustering coefficient: {}.'.format(stat_dict['C']))

    stat_string.append('Mean cluster length (arithmetic): {}.'.format(stat_dict['a_mean_size']))

    stat_string.append('Mean cluster length (harmonic): {}.'.format(stat_dict['h_mean_size']))

    stat_string.append('Number of clusters: {}.'.format(stat_dict['clusters']))

    stat_string.append('Tuples gained through merging: {}.'.format(stat_dict['pipe_gain']))

    stat_string.append('Sense inventory:')

    for hub in stat_dict['hubs'].keys():

        stat_string.append(' -> {}: {}.'.format(hub, ", ".join(stat_dict['hubs'][hub])))

    print('\n[A] '+'\n[A] '.join(stat_string)+'\n')

    with open('../baseline/statistics.txt', 'a') as stat_file:

        stat_file.write('\n '.join(stat_string)+'\n\n')

    write_header = not os.path.exists('.statistics.tsv')

    with open('../baseline/.statistics.tsv', 'a') as stat_file:

        if write_header:

            stat_file.write('\t'.join(key_list)+'\n')

        stat_file.write('\t'.join([str(stat_dict[key]) for key in key_list])+'\n')

def global_clustering_coefficient(graph: nx.Graph) -> float:

    """Calculates global clustering coefficient from graph.

    Iterates over every node and calculates the global coefficient as a mean

    of every local clustering coefficient. 

    Args:

        graph: Undirected graph.

    Returns:

        Global coefficient.

    """

    local_coefficient_list = list()

    for node in graph.nodes:

        neighbor_list = graph.adj[node]

        neighbor_edge_list = [(x,y) for x in neighbor_list 

                              for y in neighbor_list if x<y]

        if len(neighbor_edge_list) == 0:

            local_coefficient_list.append(0)

        else:

            edge_count = 0

            for x,y in neighbor_edge_list:

                if graph.has_edge(x,y):

                    edge_count += 1

            local_coefficient_list.append(edge_count/len(neighbor_edge_list))

    return np.mean(local_coefficient_list)

def characteristic_path_length(graph: nx.Graph) -> float:

    """Calculates characteristic path length from graph.

    Iterates over every node tuple and calculates the shortest path between them.

    The average path length is returned. Tuples without path are ignored.

    Args:

        graph: Undirected graph.

    Returns:

        Global coefficient.

    """

    path_length_list = list()

    path_list = [(x,y) for x in graph.nodes for y in graph.nodes if x<y]

    for path in path_list:

        if nx.has_path(graph,*path):

            shortest_path = nx.shortest_path(graph,*path)

            path_length_list.append(len(shortest_path))

    return np.mean(path_length_list)

def main(topic_id: int, topic_name: str, result_dict: dict) -> None:

    """Calls induction and disambiguation functions, performs main task.

    The task is to both induce senses and match search results to them. This

    function calls in much the same way induce() and disambiguate_mst() to 

    perform these sub tasks. The result is then written to the output directory

    specified in config.py.

    Args:

        topic_id: Index of topic in topics.txt.

        topic_name: Target string.

        result_dict: Dictionary with topic_id as key and list of search queries

            (from results.txt) as values.

    """

    if topic_name in [output_file_name.replace('.absinth', '') 

                      for output_file_name in os.listdir(config.base_out)]:

        return None

    else:

        print('[a]', 'Inducing word senses for {}.'.format(topic_name))

        graph, root_hub_list, stat_dict = induce(topic_name, result_dict[topic_id])

        stat_dict['L'] = characteristic_path_length(graph)

        stat_dict['C'] = global_clustering_coefficient(graph)

        edge_count = len(graph.edges)

        node_count = len(graph.nodes)

        mean_degree = edge_count/node_count

        stat_dict['L_rand'] = np.log(node_count)/np.log(mean_degree)

        stat_dict['C_rand'] = 2 * mean_degree/node_count

        print('[a]', 'Disambiguating results.\t('+topic_name+')')