Add visualisation and corpus reform.

import spacy # for nlp

from multiprocessing import Pool

import random

import matplotlib.pyplot as plt

nlp = spacy.load('en') # standard english nlp

#counts occurences of nodes and cooccurrences

def frequencies(corpus_path, target):

def frequencies(corpus_path, target, results):

    random.seed(1)

    stop_words = set(stopwords.words('english') + config.stop_words)

    allowed_tags = config.allowed_tags

    min_context_size = config.min_context_size

    max_nodes = config.max_nodes

    max_edges = config.max_edges

    node_freq = dict() #counts (potential) nodes

    edge_freq = dict() #counts (potential) edges

    results = [r.replace('<b>', '').replace('</b>', '').replace(r'\\', '').strip() for r in results]

    node_freq, edge_freq = process_file(results, target) #initialises frequencies with counts from results

    s_target = target.replace('_', ' ') #target word with spaces

    files = [corpus_path + f for f in os.listdir(corpus_path)] #file names of corpus files

    random.shuffle(files)

    i = 0 #for update print statements

    for f in files:

        #checks maximum node values

        if len(node_freq) > max_nodes:

            print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq))+'\t('+target+')')

            return node_freq, edge_freq

        #checks maximum edge values

        if len(edge_freq) > max_edges:

            print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq))+'\t('+target+')')

            return node_freq, edge_freq

        with open(f, 'r') as lines: #parses single file

            node_freq, edge_freq = process_file(lines, target, node_freq, edge_freq)

        i += 1

    #update print

    print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq))+'\t('+target+')')

    return node_freq, edge_freq

def process_file(lines, target, node_freq=None, edge_freq=None):

    if node_freq is None:

        node_freq = dict()

    if edge_freq is None:

        edge_freq = dict()

    s_target = target.replace('_', ' ') #target word with spaces

    stop_words = set(stopwords.words('english') + config.stop_words)

    allowed_tags = config.allowed_tags

    min_context_size = config.min_context_size

    try:

        for line in lines: #parses single paragraph

        pass

        #print('Failed to decode:', f)              

        i += 1

    #update print

    print('[a] 100%\tNodes: {}\tEdges: {}.'.format(len(node_freq), len(edge_freq))+'\t('+target+')')

    return node_freq, edge_freq

#build graph from frequency dictionaries

def build_graph(node_freq, edge_freq):

# Basically Word Sense Disambiguation, matches context to sense

def disambiguate(mst, hubs, contexts, target=""):

def disambiguate(mst, hubs, contexts, target):

    target = target.replace('_', ' ')

    T = mst #minimum spanning tree

        return {0:[i for i in range(1, len(C)+1)]}

    idx = 0

    for c in C:

        idx = C.index(c) + 1 #index based on position in list

        idx += 1 #index based on position in list

        doc = nlp(c) #parsed context

        texts = [tok.text for tok in doc] #tokens

    return mapping_dict

def draw_graph(G, name):

    nx.draw_networkx(G,pos=nx.spring_layout(G), with_labels=True, node_size=40, font_size=9, node_color='#2D98DA')

    plt.savefig('../figures/'+name+'.png', dpi=200, bbox_inches='tight')

    plt.clf()

# our main function, here the main stepps for word sense induction are called

def WSI(topic_id, topic_name, results):

    #counts occurences of single words, as well as cooccurrences, saves it in dictionary

    print('[a]', 'Counting nodes and edges.\t('+old_target+')')

    node_freq, edge_freq = frequencies(corpus_path, target)

    node_freq, edge_freq = frequencies(corpus_path, target, results[topic_id])

    #builds graph from these dictionaries, also applies multiple filters

    print('[a]', 'Building graph.\t('+old_target+')')

    G = build_graph(node_freq, edge_freq)

    draw_graph(G, topic_name.strip()+'_g')

    out_buffer += '[A] Nodes: {}\tEdges: {}\n'.format(str(len(G.nodes)), str(len(G.edges)))

    #finds root hubs (senses) within the graph + more filters for these

    #performs minimum_spanning_tree algorithm on graph

    print('[a]', 'Building minimum spanning tree.\t('+old_target+')')

    T = components(G, H, target)

    draw_graph(T, topic_name.strip()+'_t')

    #matches senses to clusters

    print('[a]', 'Disambiguating results.\t('+old_target+')')

            topics[l[0]] = l[1]

    # multiprocessing

    with Pool(4) as pool:

    with Pool(5) as pool:

        # calls WSI() for for topics at a time

        pool.starmap(WSI, [(key, value, results) for key,value in topics.items()])