Commit d01631b5 authored by Victor Zimmermann's avatar Victor Zimmermann
Browse files

Merge branch 'colour_dist' into 'master' 

Better Statistics, bugfixes.

See merge request zimmermann/absinth!2
parents 42d66c63 6ee3d178

src/absinth.py

+51 −12
Original line number Diff line number Diff line
@@ -32,9 +32,11 @@ import os # for reading files 
import pprint

import re

import spacy # for nlp

import time



from multiprocessing import Pool

from copy import deepcopy

from multiprocessing import Pool

from scipy import stats 





nlp = spacy.load('en') # standard english nlp
@@ -499,13 +501,6 @@ def induce(topic_name: str, result_list: list) -> (nx.Graph, list, dict): 
    

    stat_dict = dict()

    

    if topic_name in [output_file_name.replace('.absinth', '') 

                      for output_file_name in os.listdir(config.output)]:

        

        return None

    

    else:

    

    stat_dict['target'] = topic_name

        

    #in topics longer than two words, the leading 'the' can generally be removed without changing the sense
@@ -692,9 +687,10 @@ def disambiguate_colour(graph: nx.Graph, root_hub_list: list, context_list: list 
                                    coloured_graph[sub_from][sub_to]['weight']

                

                            score[root_hub_idx] += (1/(1+total_weight)) \

                             * colour_graph.node[text]['dist'][root_hub_idx]

                             * coloured_graph.node[text]['dist'][root_hub_idx]

                    

                        else:

                            

                            pass

            

            else:
@@ -799,6 +795,34 @@ def disambiguate_mst(graph: nx.Graph, root_hub_list: list, 
    return mapping_dict





def print_stats(stat_dict: dict) -> None:

    """Prints various statistics and logs them to file.

    

    Args:

        stat_dict: Dictionary with various statistics.

    

    """

    

    stat_string = []

    

    ts = time.gmtime()

    

    stat_string.append('[A] Topic:\t{}.'.format(stat_dict['target']))

    stat_string.append('[A] Processed {} at {}.'.format(time.strftime("%Y-%m-%d", ts),time.strftime("%H:%M:%S", ts)))

    stat_string.append('[A] Nodes: {}\tEdges: {}.'.format(stat_dict['node count'],stat_dict['edge count']))

    stat_string.append('[A] Mean cluster length (harmonic):\t{}.'.format(stat_dict['hmean_cluster_length']))

    stat_string.append('[A] Mean cluster length (arithmetic):\t{}.'.format(stat_dict['mean_cluster_length']))

    stat_string.append('[A] Number of clusters: {}.'.format(stat_dict['cluster_count']))

    stat_string.append('[A] Tuples gained through merging: {}.'.format(stat_dict['merge_gain']))

    stat_string.append('[A] Sense inventory:')

    for hub in stat_dict['hubs'].keys():

        stat_string.append('[A] {}:\t{}.'.format(hub, ", ".join(stat_dict['hubs'][hub])))

    

    with open('stats.txt', 'a') as stat_file:

        stat_file.write('\n'.join(stat_string)+'\n\n')

        print('\n'+'\n'.join(stat_string)+'\n')

        



def main(topic_id: int, topic_name: str, result_dict: dict) -> None:

    """Calls induction and disambiguation functions, performs main task.
@@ -815,7 +839,14 @@ def main(topic_id: int, topic_name: str, result_dict: dict) -> None: 
            

    """

    

    if topic_name in [output_file_name.replace('.absinth', '') 

                      for output_file_name in os.listdir(config.output)]:

        return None

    

    else:

        

        print('[a]', 'Inducing word senses for {}.'.format(topic_name))

        

        graph, root_hub_list, stat_dict = induce(topic_name, result_dict[topic_id])

        

        colour_rank = config.colour_rank
@@ -881,6 +912,7 @@ def main(topic_id: int, topic_name: str, result_dict: dict) -> None: 
                cluster_count += 1

                cluster_length_list.append(cluster_length)

                

        stat_dict['hmean_cluster_length'] = stats.hmean(cluster_length_list)

        stat_dict['mean_cluster_length'] = np.mean(cluster_length_list)

        stat_dict['cluster_count'] = cluster_count
@@ -899,7 +931,8 @@ def main(topic_id: int, topic_name: str, result_dict: dict) -> None: 
                                                        topic_id, result_id)

                    output_file.write(output_line)

                    

    pprint.pprint(stat_dict)

        print_stats(stat_dict)
@@ -921,11 +954,17 @@ if __name__ == '__main__': 
    

    # Enables manual setting of process count.

    if '-p' in sys.argv:

        

        process_count = int(sys.argv[sys.argv.index('-p') + 1])

    else:

        process_count = 1

        

        with Pool(process_count) as pool:

            

            parameter_list = [(topic_id, topic_name, result_dict)

                              for topic_id,topic_name in topic_dict.items()]

            pool.starmap(main, sorted(parameter_list)) #determineate function



    else:

        

        for topic_id, topic_name in sorted(topic_dict.items()):

            main(topic_id, topic_name, result_dict)

src/config.py

+1 −1
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ Methods labeled with 0 are ignored. 
At least one method must be given a value != 0.

'''

resolve_conflicts = False #not yet implemented

mst_rank = 0

mst_rank = 2

colour_rank = 1



'''