added code and example dicts

from sklearn.metrics import precision_score,recall_score,f1_score,precision_recall_fscore_support

import numpy as np

from scipy.stats import pearsonr

def make_multi_label_mat(true,preds,nclasses=18):

    A=[]

    B=[]

    for i in range(nclasses):

        #print(nclasses)

        A+=np.argmax(true[i],axis=1).tolist()

        B+=np.argmax(preds[i],axis=1).tolist()

    AG=np.array(A).reshape((nclasses,len(preds[0]))).T

    AP=np.array(B).reshape((nclasses,len(preds[0]))).T

    return AG,AP

def evaluate(true,preds,nclasses=18):

    AG,AP = make_multi_label_mat(true,preds,nclasses)

    def f1(a,b):

        return (2*a*b)/(a+b)

    ma=f1_score(AG,AP,average='macro')

    ma2=f1(precision_score(AG,AP,average='macro'),recall_score(AG,AP,average='macro'))

    mi=f1(precision_score(AG,AP,average='micro'),recall_score(AG,AP,average='micro'))

    return [ma,ma2,mi]

def evaluate_regression(true,preds,nclasses=18):

    AG,AP = [np.array(t).flatten() for t in true],[np.array(p).flatten() for p in preds]

    ps = [pearsonr(AG[i],AP[i])[1] for i in range(len(AP))]

    rhos = [pearsonr(AG[i],AP[i])[0] for i in range(len(AP))]

    return [np.mean(np.nan_to_num(ps)),np.mean(np.nan_to_num(rhos))]

def extensive_evaluate(true,preds,nclasses=18):

    AG,AP = make_multi_label_mat(true,preds,nclasses)

    return precision_recall_fscore_support(AG,AP)

def extensive_evaluate_regression(true,preds,nclasses=18):

    AG,AP = [np.array(t).flatten() for t in true],[np.array(p).flatten() for p in preds]

    ps = [pearsonr(AG[i],AP[i])[1] for i in range(len(AP))]

    rhos = [pearsonr(AG[i],AP[i])[0] for i in range(len(AP))]

    return [np.mean(np.nan_to_num(ps)),np.mean(np.nan_to_num(rhos)),np.nan_to_num(np.array(rhos)),np.nan_to_num(np.array(ps))]

import os

import argparse

from numpy.random import seed

from eval_helpers import evaluate, extensive_evaluate,extensive_evaluate_regression,evaluate_regression

import config

from data_setup import DataSet

from model import ModelConstructor

parser = argparse.ArgumentParser(description="Argument Parser for Proto Labeling")

parser.add_argument("PATH_DATASET",type=str,help="datasetpath")

parser.add_argument("runid",type=str,help="runid")

args = parser.parse_args()

seed(int(args.runid))

#prepare data

ds = DataSet(args.PATH_DATASET,LENGTH=config.LENGTH,AUXREG=config.AUXREG,SOFTMAX=config.SOFTMAX,PRETRAINED=config.PRETRAINED)

ds.prepare()

NCLASSES = ds.NCLASSES

trainxwords,devxwords,testxwords,trainxpointers,devxpointers,testxpointers,trainy,devy,testy = ds.make_data()

from model import ModelConstructor

#get model instance

model = ModelConstructor(HIER=config.HIER,LENGTH=config.LENGTH,AUXREG=config.AUXREG,SOFTMAX=config.SOFTMAX,ATT=config.ATT,MARKER_1=config.MARKER_1,MARKER_2=config.MARKER_2,EMBEDDING_DIM=config.EMBEDDING_DIM,NPOINTERS=4,LSTMUNITS=config.LSTMUNITS,NCLASSES=NCLASSES).make_model()

bestdev=0.0

for epoch in range(10):

    print("EPOCH",epoch)

    model.fit([trainxwords,trainxpointers],[ys for ys in trainy],epochs=1,batch_size=4,verbose=2)

    xpred = model.predict([devxwords,devxpointers])

    if config.SOFTMAX:

        ev=evaluate(devy[:NCLASSES],xpred[:NCLASSES],NCLASSES)

    else:

        ev=evaluate_regression(devy[:NCLASSES],xpred[:NCLASSES],NCLASSES)

    #if score on dev better than previous epoch, save model ev[1] is macro F1

    if ev[1] > bestdev:

        print("f1/pr on dev:",ev[1],"improvement:",ev[1]-bestdev,"saveing model...")

        if config.SOFTMAX:

            print("f1/pr on dev,extensive:",extensive_evaluate(devy[:NCLASSES],xpred[:NCLASSES],NCLASSES))

        else:

            print("f1/pr on dev,extensive:",extensive_evaluate_regression(devy[:NCLASSES],xpred[:NCLASSES],NCLASSES))

        bestdev=ev[1]

        model.save('models/'+args.runid+'my_model.h5')

        xpredt = model.predict([testxwords,testxpointers])

#print test predictions from best dev epoch this is needed in final ensemble vote

print("predictions:",[x.tolist() for x in xpredt])

print("gold:",[y.tolist() for y in testy])

#print scores on test from this model

if config.SOFTMAX:

    print(extensive_evaluate(testy[:NCLASSES],xpredt[:NCLASSES],NCLASSES))

else:

    print(extensive_evaluate_regression(testy[:NCLASSES],xpredt[:NCLASSES],NCLASSES))

#!/bin/bash

for i in `seq 0 49`;

    do

        #this can be parallelized, easy by setting "&" at the end of the line or better on a specialized cluster

        #it will write max(i) logs which contain the test votes of the single models

        python -u proto_model_bert.py resources/exampledict_spr1.json $i > bertbaselogs/$i-ml.log 

    done