added code

from __future__ import print_function

import os

os.environ["CUDA_VISIBLE_DEVICES"] = "0"

from keras_multi_head import MultiHeadAttention

from keras_self_attention import SeqSelfAttention, SeqWeightedAttention

from sklearn.utils.class_weight import compute_sample_weight

from keras.constraints import NonNeg

import sys

import numpy as np

from random import shuffle

from keras.layers import Dropout

from sklearn.metrics import accuracy_score,matthews_corrcoef,f1_score,precision_score,recall_score

#import ast

from keras import backend as K

from keras.models import Model

from keras.layers import Embedding, Flatten, Input, merge, Concatenate,LSTM,Bidirectional,Add,Multiply,Activation,Lambda

from keras.layers.core import Dense

from keras.optimizers import Adam

from keras.models import load_model

from keras.layers.embeddings import Embedding

import metrics

import argparse

import os

from flair.embeddings import ELMoEmbeddings, Sentence, BertEmbeddings 

import json

import data_helpers as dh

import argparse

parser = argparse.ArgumentParser()

parser.add_argument("DATA")

parser.add_argument("EMB")

parser.add_argument("RUNID")

parser.add_argument("-maxlen",nargs='?',default=50,type=int)

parser.add_argument("-epochs",nargs='?',default=25,type=int)

parser.add_argument('-sup_connect',

                            nargs='?', default="Yes+,")

parser.add_argument('-att_connect',

                            nargs='?', default="No+,")

parser.add_argument("--context", help="use context if available",

                            action="store_true")

parser.add_argument("--use_markers", help="use coefficients",

                            action="store_true")

args = parser.parse_args()

print(args)

logstring = "logs/data={}-emb={}-runid={}-maxlen={}-sup_connect={}-att_connect={}-context={}-markers={}".format(args.DATA,args.EMB,args.RUNID,args.maxlen,args.sup_connect,args.att_connect,args.context,args.use_markers)

modelstring = "models/data={}-emb={}-runid={}-maxlen={}-sup_connect={}-att_connect={}-context={}-markers={}".format(args.DATA,args.EMB,args.RUNID,args.maxlen,args.sup_connect,args.att_connect,args.context,args.use_markers)

print(logstring)

#the identity loss ensures that the prediction score (rank loss) is reduced at training time

def identity_loss(y_true, y_pred):

    return K.mean(y_pred - 0 * y_true)

#the rank loss gets used in the final lambda layer and the model outputs the loss as "prediction"

# training time: prediction of the model is ranking loss ("artificial last layer")

# testing/inference time: artificial lambda layer is removed, prediction of the model is input to rank_loss

def rank_loss(X):

    positive_item_score, negative_item_score = X

    margin=1.0

    loss =  K.sum(K.maximum(0.0,margin-positive_item_score + negative_item_score))

    return loss

#generate marker embedding

if "bert" == args.EMB:

    markembedding= Embedding(7,1024,trainable=True,embeddings_constraint=NonNeg(),embeddings_initializer="ones")

    c_text_INPUT = Input((args.maxlen,1024  ), name='positive_input')

    f_text_INPUT = Input((args.maxlen,1024 ), name='negative_input')

    markers_INPUT = Input((args.maxlen, ),name='input_markers')

elif "elmo" == args.EMB:

    markembedding= Embedding(7,768,trainable=True,embeddings_constraint=NonNeg(),embeddings_initializer="ones")

    c_text_INPUT = Input((args.maxlen,768  ), name='positive_input')

    f_text_INPUT = Input((args.maxlen,768 ), name='negative_input')

    markers_INPUT = Input((args.maxlen, ),name='input_markers')

#LSTM

lstm = Bidirectional(LSTM(256,return_sequences=True)) 

#multi headed self attention

sa = MultiHeadAttention(

            head_num=4,

                name='Multi-Head2',

                )

#output layer for providing one score of "plausibility"

shared_dense2 = Dense(1,activation="selu",name="l")

#merge ops

mult=Multiply()

add=Add()

#we process token sequences of correct adn false reading with the same shared layers

#MARKERS = True

c_text_embedded = c_text_INPUT

f_text_embedded = f_text_INPUT

if args.use_markers:

    ma_emb = markembedding(markers_INPUT)

    c_text_embedded = mult([ma_emb,c_text_INPUT])

    f_text_embedded = mult([ma_emb,f_text_INPUT])

final_matrix_repr_c = sa(lstm(c_text_embedded))

final_matrix_repr_f = sa(lstm(f_text_embedded))

#op to merge seq of vectors to single vector

wa = SeqWeightedAttention()

#apply op

final_vect_repr_c = wa(final_matrix_repr_c)

final_vect_repr_f = wa(final_matrix_repr_f)

# we end up with two plausibility scores, one for each reading

c_score = shared_dense2(final_vect_repr_c)

f_score = shared_dense2(final_vect_repr_f)

#ranking loss

loss = Lambda(rank_loss, output_shape=(1, ))([c_score, f_score]) 

model = Model(

    inputs=[c_text_INPUT, f_text_INPUT, markers_INPUT],

    outputs=loss)

model.compile(loss=identity_loss, optimizer=Adam(lr=0.001))

print(model.summary())

import pre_pro 

#preprocess data, use BERT/Elmo for fea extraction

pp = pre_pro.PreProcessor(DATA=args.DATA

                         ,EMB=args.EMB

                         ,context=args.context

                         ,sup_connect=args.sup_connect

                         ,att_connect=args.att_connect

                         ,RUNID=args.RUNID

                         ,maxlen=args.maxlen)

#train dev test split

X,valX,testX,trainlabels,devlabels,testlabels = pp.get_preprocessed_data()

epc = 0

epcb = 0

bestsc = 0

bs=0

bestaccval = 0.0

#every sample has the same weight

sample_weight = np.ones(len(X["positive_input"]))

#we save the last 5 best model ckpts

fivebestaccs=[0.0]*5

outstring = ""

for ep in range(args.epochs):

    epc+=1

    print("epoch",ep) 

    #as target we feed an array of 1s - target does not matter in training, loss is computed based on the two input instances

    model.fit(X,np.ones(len(X["positive_input"])),batch_size=64,verbose=2,epochs=1,sample_weight=sample_weight)

    # construct new model, since output is not used for loss but we are actually interested in the prediction of plausibiltiy

    model2 = Model(inputs=[c_text_INPUT, f_text_INPUT, markers_INPUT], outputs=[loss,c_score,f_score])

    model2.set_weights(model.get_weights())  

    #make validation predictions

    accval = metrics.get_classif_report(model2,valX,devlabels)

    print("validation accuracy",accval)

    print("----")

    #if new validation score better than old, drop a ckpt

    fivebestaccs=list(sorted(fivebestaccs))

    print(fivebestaccs)

    if any([accval > x for x in fivebestaccs]):

        print("dropping checkpoint")

        model.save_weights(modelstring+"-DEVACC:"+str(accval)+'-model.h5')

        for i,e in enumerate(fivebestaccs):

            if accval > e:

                fivebestaccs[i] = accval

            break

    outstring+=str(fivebestaccs)+"\n"

# we find the five best model ckpts on dev

modelfiles = list(sorted(['models/'+m for m in os.listdir('models') if modelstring in 'models/'+m],key=lambda string:float(string.split("DEVACC:")[1].split("-")[0])))

print(modelfiles)

# but we only use the best ckpt on dev to predict test (later we also ensemble the predictions of the five best ckpts)

print("loading best model weights on dev")

model2 = Model(inputs=[c_text_INPUT, f_text_INPUT, markers_INPUT], outputs=[loss,c_score,f_score])

model2.load_weights(modelfiles[-1])

outstring+="best on dev, test predictions: "+str(metrics.predict_nli(model2,testX).T.tolist())+"\n"

print("loading best model weights on dev, 5 checkpoint avg")

model2 = Model(inputs=[c_text_INPUT, f_text_INPUT, markers_INPUT], outputs=[loss,c_score,f_score])

modelfiles_best=modelfiles[-5:]

[os.remove(mf) for mf in modelfiles if mf not in modelfiles_best]

print(modelfiles)

scores = np.zeros((len(testX["positive_input"]),2))

ls = []

for mf in modelfiles_best:

    model2.load_weights(mf)

    preds = metrics.predict_nli(model2,testX).T

    scores += preds

    ls.append(preds.tolist())

#print(metrics.get_classif_report_scores(scores,testX,testlabels))

outstring+="best 5 on dev, test predictions: "+str(ls)+"\n"

# we write the ground truth so we can evaluate against it

outstring+="ground truth: "+str(testlabels)+"\n"

with open(logstring+".predictions","w") as f:

    f.write(outstring)

#data identifiers and paths

DAT = {"ESSAY": {"train":"../data/essay/train.txt","dev":"../data/essay/dev.txt", "test":"../data/essay/test.txt", "ctx":None }}

import sys

import numpy as np

from nltk.tokenize import word_tokenize

from scipy.sparse import hstack

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.metrics import confusion_matrix

import json

import ast

from sklearn.metrics import f1_score,classification_report

from copy import deepcopy

from sklearn.svm import LinearSVC

from data_config import DAT

def create_data(DATA):

    train_words = []

    dev_words = []

    test_words = []

    train_labels = []

    dev_labels = []

    test_labels = []

    if "dev" in DAT[DATA]:

        PTRAIN = DAT[DATA]["train"]

        PDEV = DAT[DATA]["dev"]

        PTEST = DAT[DATA]["test"]

    if "dev" in DAT[DATA]:

        with open(PTRAIN,encoding="utf-8") as f:

            lines = [l.split("\t") for l in f.read().split("\n") if "\t" in l]

            for i,l in enumerate(lines):

                sw = word_tokenize(l[1])

                tw = word_tokenize(l[2])

                bw = [t for t in sw if t in tw]

                train_labels.append(l[-1])

                train_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in bw])

                #print(l[5])

                """"

                paths = ast.literal_eval(l[5])

                for j,pa in enumerate(paths):

                    pw = pa.split()

                    train_words[i]+=["PATH"+str(j)+":" + t for t in pw]

                """

                if i %100 == 0:

                    print(i,"train examples created")

        with open(PDEV,encoding="utf-8") as f:

            lines = [l.split("\t") for l in f.read().split("\n") if "\t" in l]

            for i,l in enumerate(lines):

                sw = word_tokenize(l[1])

                tw = word_tokenize(l[2])

                bw = [t for t in sw if t in tw]

                dev_labels.append(l[-1])

                dev_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in bw])

                """

                paths = ast.literal_eval(l[5])

                for j,pa in enumerate(paths):

                    pw =pa.split()

                    dev_words[i]+=["PATH"+str(j)+":" + t for t in pw]

                """

    #just in case you don't have dev set, the we split of dev from train

    else:

        attacks=0

        supports=0

        trainidx=0

        devidx=0

        with open(PTRAIN) as f:

            lines = [l.split("\t") for l in f.read().split("\n") if "\t" in l]

            for i,l in enumerate(lines):

                sw = word_tokenize(l[1])

                tw = word_tokenize(l[2])

                bw = [t for t in sw if t in tw]

                if l[-1] == "support" and supports <=46:

                    supports+=1

                    dev_labels.append(l[-1])

                    dev_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in sw if t in tw])

                    """

                    paths = ast.literal_eval(l[5])

                    for j,pa in enumerate(paths):

                        pw = pa.split()

                        dev_words[devidx]+=["PATH"+str(j)+":" + t for t in pw]

                    """

                    devidx+=1

                elif l[-1] == "attack" and attacks <= 20:

                    attacks+=1

                    dev_labels.append(l[-1])

                    dev_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in sw if t in tw])

                    """

                    paths = ast.literal_eval(l[5])

                    for j,pa in enumerate(paths):

                        pw = pa.split()

                        dev_words[devidx]+=["PATH"+str(j)+":" + t for t in pw]

                    """

                    devidx+=1

                else:

                    train_labels.append(l[-1])

                    train_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in sw if t in tw])

                    """

                    paths = ast.literal_eval(l[5])

                    for j,pa in enumerate(paths):

                        pw = pa.split()

                        train_words[trainidx]+=["PATH"+str(j)+":" + t for t in pw]

                    """

                    trainidx+=1

    with open(PTEST,encoding="utf-8") as f:

        lines = [l.split("\t") for l in f.read().split("\n") if "\t"  in l]

        for i,l in enumerate(lines):

            sw = word_tokenize(l[1])

            tw = word_tokenize(l[2])

            bw = [t for t in sw if t in tw]

            test_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in bw])

            test_labels.append(l[-1])

            """

            paths = ast.literal_eval(l[5])

            for j,pa in enumerate(paths):

                pw = pa.split()

                test_words[i]+=["PATH"+str(j)+":" + t for t in pw]

            """

    for i,la in enumerate(train_labels):

        if "support" in la:

            train_labels[i] = "support"

        elif "attack" in la:

            train_labels[i] = "attack"

        else:

            print("forbidden label",la)

            sys.exit(1)

    for i,la in enumerate(dev_labels):

        if "support" in la:

            dev_labels[i] = "support"

        elif "attack" in la:

            dev_labels[i] = "attack"

        else:

            print("forbidden label",la)

            sys.exit(1)

    for i,la in enumerate(test_labels):

        if "support" in la:

            test_labels[i] = "support"

        elif "attack" in la:

            test_labels[i] = "attack"

        else:

            print("forbidden label",la)

            sys.exit(1)

    return train_words,dev_words,test_words,train_labels,dev_labels,test_labels

def get_context(PC=""):

    dic={}

    with open(PC,"r",encoding="utf-8") as f:

        lines = [l.split("\t") for l in f.read().split("\n") if "\t" in l]

        for l in lines:

            key = l[0].split("_",1)[1]

            dic[key] = {"cs":word_tokenize(l[1])}

            dic[key]["ct"] = word_tokenize(l[2])

    return dic

def create_data_context(DATA):

    train_words = []

    dev_words = []

    test_words = []

    train_context = []

    dev_context = []

    test_context = []

    train_labels = []

    dev_labels = []

    test_labels = []

    stuc = get_context(DAT[DATA]["ctx"])

    if "dev" in DAT[DATA]:

        PTRAIN = DAT[DATA]["train"]

        PDEV = DAT[DATA]["dev"]

        PTEST = DAT[DATA]["test"]

    with open(PTRAIN,encoding="utf-8") as f:

        lines = [l.split("\t") for l in f.read().split("\n") if "\t" in l]

        for i,l in enumerate(lines):

            sw = word_tokenize(l[1])

            tw = word_tokenize(l[2])

            bw = [t for t in sw if t in tw]

            train_labels.append(l[-1])

            train_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in bw])

            train_context.append(["SOURCE:"+t for t in stuc[l[0]]["cs"]]+["TARGET:"+t for t in stuc[l[0]]["ct"]])

            """

            paths = ast.literal_eval(l[5])

            for j,pa in enumerate(paths):

                pw = pa.split()

                train_words[i]+=["PATH"+str(j)+":" + t for t in pw]

            """

            if i %100 == 0:

                print(i,"train examples created")

    with open(PDEV,encoding="utf-8") as f:

        lines = [l.split("\t") for l in f.read().split("\n") if "\t" in l]

        for i,l in enumerate(lines):

            sw = word_tokenize(l[1])

            tw = word_tokenize(l[2])

            bw = [t for t in sw if t in tw]

            dev_labels.append(l[-1])

            dev_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in bw])

            dev_context.append(["SOURCE:"+t for t in stuc[l[0]]["cs"]]+["TARGET:"+t for t in stuc[l[0]]["ct"]])

            """

            paths = ast.literal_eval(l[5])

            for j,pa in enumerate(paths):

                pw =pa.split()

                dev_words[i]+=["PATH"+str(j)+":" + t for t in pw]

            """

    with open(PTEST,encoding="utf-8") as f:

        lines = [l.split("\t") for l in f.read().split("\n") if "\t"  in l]

        for i,l in enumerate(lines):

            idx = l[0]

            sw = word_tokenize(l[1])

            tw = word_tokenize(l[2])

            bw = [t for t in sw if t in tw]

            test_words.append(["SOURCE:"+t for t in sw]+["TARGET:"+t for t in tw]+["BOTH:"+t for t in bw])

            test_labels.append(l[-1])

            test_context.append(["SOURCE:"+t for t in stuc[l[0]]["cs"]]+["TARGET:"+t for t in stuc[l[0]]["ct"]])

            """

            paths = ast.literal_eval(l[5])

            for j,pa in enumerate(paths):

                pw = pa.split()

                test_words[i]+=["PATH"+str(j)+":" + t for t in pw]

            """

    for i,la in enumerate(train_labels):

        if "support" in la:

            train_labels[i] = "support"

        elif "attack" in la:

            train_labels[i] = "attack"

        else:

            print("forbidden label",la)

            sys.exit(1)

    for i,la in enumerate(dev_labels):

        if "support" in la:

            dev_labels[i] = "support"

        elif "attack" in la:

            dev_labels[i] = "attack"

        else:

            print("forbidden label",la)

            sys.exit(1)

    for i,la in enumerate(test_labels):

        if "support" in la:

            test_labels[i] = "support"

        elif "attack" in la:

            test_labels[i] = "attack"

        else:

            print("forbidden label",la)