Redone output, shuffle corpus, disambiguation output is now a dict

import config

import spacy # for nlp

from multiprocessing import Pool

import random

nlp = spacy.load('en') # standard english nlp

#counts occurences of nodes and cooccurrences

def frequencies(corpus_path, target):

    random.seed(1)

    stop_words = set(stopwords.words('english') + config.stop_words)

    allowed_tags = config.allowed_tags

    min_context_size = config.min_context_size

    node_freq = dict() #counts (potential) nodes

    edge_freq = dict() #counts (potential) edges

    files = [corpus_path + f for f in os.listdir(corpus_path)] #file names of corpus files

    s_target = target.replace('_', ' ') #target word with spaces

    files = [corpus_path + f for f in os.listdir(corpus_path)] #file names of corpus files

    random.shuffle(files)

    i = 0 #for update print statements

    for f in files:

        if i % int(len(files)/10) == 0: #prints update after every 10th of the corpus is parsed

        if i % int(len(files)/11) == 0: #prints update after every 10th of the corpus is parsed

            file_ratio = i/len(files[:])

            max_node_ratio = len(node_freq)/max_nodes

            #uses the ratio closest to 100%.

            percentage = int((max(ratios))*100)

            print('[a] ~{:02d}%\tNodes: {}\tEdges: {}.'.format(percentage, len(node_freq), len(edge_freq)), target)

            print('[a] ~{:02d}%\tNodes: {}\tEdges: {}.'.format(percentage, len(node_freq), len(edge_freq))+'\t('+target+')')

        #checks maximum node values

        if len(node_freq) > max_nodes:

    C = [c.lower().strip().replace(target, '') for c in contexts] #cleaned up contexts

    score_dict = dict() #memoisation for scores

    result = list() #output of function

    for c in C:

        idx = C.index(c) + 1 #index based on position in list

    mapping_dict = {topic:[] for topic in range(1,len(H)+1)} #output of function

    #if no sense is found for a target word, we should assume that there only is one sense

    if len(H) == 0: 

            result.append((1, idx, 0))

        return {0:[i for i in range(1, len(C)+1)]}

        else:

    for c in C:

        idx = C.index(c) + 1 #index based on position in list

        doc = nlp(c) #parsed context

        texts = [tok.text for tok in doc] #tokens

            argmax_score = np.argmax(scores)

            #clusters begin at 1

                result.append((argmax_score + 1, idx))

            mapping_dict[argmax_score + 1].append(idx)

    return result

    return mapping_dict

# our main function, here the main stepps for word sense induction are called

    f.write('subTopicID\tresultID\n')

    #counts occurences of single words, as well as cooccurrences, saves it in dictionary

    print('[a]', 'Counting nodes and edges.', old_target)

    print('[a]', 'Counting nodes and edges.\t('+old_target+')')

    node_freq, edge_freq = frequencies(corpus_path, target)

    out_buffer += '[A] Nodes: {}\tEdges: {}\n'.format(str(len(node_freq)), str(len(edge_freq)))

    #builds graph from these dictionaries, also applies multiple filters

    print('[a]', 'Building graph.', old_target)

    print('[a]', 'Building graph.\t('+old_target+')')

    G = build_graph(node_freq, edge_freq)

    #finds root hubs (senses) within the graph + more filters for these

    print('[a]', 'Collecting root hubs.', old_target)

    print('[a]', 'Collecting root hubs.\t('+old_target+')')

    H = root_hubs(G, edge_freq)

    out_buffer += '[A] Root hubs:\n'

    for h in H:

        mfn = sorted(G.adj[h], key=lambda x: edge_freq[h,x] if h < x else edge_freq[x, h], reverse=True)[:6]

        out_buffer += (' {}. {}: {}\n'.format(i, h, mfn))

        out_buffer += (' {}. {}: {}\n'.format(i, h, ', '.join(mfn)))

        i += 1

    #performs minimum_spanning_tree algorithm on graph

    print('[a]', 'Building minimum spanning tree.', old_target)

    print('[a]', 'Building minimum spanning tree.\t('+old_target+')')

    T = components(G, H, target)

    #matches senses to clusters

    print('[a]', 'Disambiguating results.', old_target)

    print('[a]', 'Disambiguating results.\t('+old_target+')')

    D = disambiguate(T, H, results[topic_id], target)

    out_buffer += ('[A] Mapping: '+ str(D) + '\n')

    out_buffer += ('[A] Mapping: \n')

    for cluster,results in D.items():

        out_buffer += (' {}. : {}\n'.format(cluster, ', '.join([str(r) for r in results])))

    #prints buffer

    print('[a]', 'Writing to file.', old_target)

    print('[a]', 'Writing to file.\t('+old_target+')')

    print(out_buffer)

    #writes clustering to file

    for d in D:

        f.write(topic_id+'.'+str(d[0])+'\t'+topic_id+'.'+str(d[1])+'\n')

    for cluster,results in D.items():

        for result in results:

            f.write(topic_id+'.'+str(cluster)+'\t'+topic_id+'.'+str(result)+'\n')

    f.close()

    # topics.txt is a list of target words

    topics = dict()

    processed_topics = [f.replace('.absinth', '') for f in os.listdir(config.output)]

    with open(data_path+'topics.txt', 'r') as topics_file:

        for line in topics_file.readlines()[1:]:

            l = line.split('\t')

            if l[1].strip() not in processed_topics:

                topics[l[0]] = l[1]

    # multiprocessing