code for preprocessing (3467e605) · Commits · opitz / SPRL

src/data_setup.py

0 → 100644

+442 −0

Original line number	Diff line number	Diff line
		import os
		from flair.data import Sentence
		from flair.embeddings import BertEmbeddings,WordEmbeddings,StackedEmbeddings
		from keras_self_attention import SeqSelfAttention, SeqWeightedAttention, ScaledDotProductAttention
		from copy import deepcopy
		from sklearn.utils.class_weight import compute_sample_weight
		from statistics import mode
		from keras import optimizers
		import sys
		import json
		import numpy as np
		from keras.layers import Embedding, Input, Dense, Bidirectional, LSTM, Concatenate, Add, Multiply,Flatten
		from keras.models import Model,load_model
		from scipy.stats import pearsonr
		from sklearn.metrics import accuracy_score,f1_score,precision_score,recall_score
		from sklearn.metrics import cohen_kappa_score
		from random import shuffle
		import pickle
		from collections import defaultdict
		import argparse
		from numpy.random import seed
		from eval_helpers import evaluate, extensive_evaluate,extensive_evaluate_regression,evaluate_regression
		#print("gpus",K.tensorflow_backend._get_available_gpus())
		from config import *
		from copy import deepcopy

		BERT="bert-base-uncased"

		class HelperFunctions():

		def __init__(self):
		return None

		def maybe_crop_and_adjust_input(self,example,length):
		"""Function to crop and adjust pointers and tokens. In place modification of input dict

		Args:
		example: a dictionary representing one data example,
		needs two keys "pointers" (string with markers separated by whitespace)
		and "sentence" (string with tokens separated by whitespace)
		length: max length

		Returns:
		None
		"""
		ptrs = example["pointers"].split(" ")
		toks= example["sentence"].split(" ")
		extracharids = [i for i in range(len(toks)) if "" in toks[i] or toks[i] == "LRB" or toks[i] == "RRB" or toks[i] == ""]
		ptrs = [ptrs[i] for i in range(len(ptrs)) if i not in extracharids]
		toks = [toks[i] for i in range(len(toks)) if i not in extracharids]
		predids = [i for i in range(len(ptrs)) if ptrs[i] == "2"]
		argids = [i for i in range(len(ptrs)) if ptrs[i] == "1"]
		maxid = max([max(predids),max(argids)])
		diff = maxid-length
		toberemoved = len(ptrs)-length
		if toberemoved > 0:
		print("INFO: pred or arg id over boundary,... adjusting")
		print("before",example["pointers"])
		print("before",example["sentence"])
		#first remove zeros inbetween
		delis=[]
		rem=toberemoved
		print(ptrs)
		#clipbefore
		for i in range(min(argids+predids)):
		if rem > 0:
		rem-=1
		delis.append(i)

		#clipafter
		for i in range(max(argids+predids)+1,len(ptrs)):
		if rem > 0:
		rem-=1
		delis.append(i)
		#clipmiddle
		for i in range(min(argids+predids),maxid):
		if rem > 0 and ptrs[i] == "0":
		rem-=1
		delis.append(i)

		ptrs=[ptrs[i] for i in range(len(ptrs)) if i not in delis]
		toks=[toks[i] for i in range(len(toks)) if i not in delis]
		ptrs=ptrs[:length]
		toks=toks[:length]
		print(delis)
		example["pointers"] = " ".join(ptrs)
		example["sentence"] = " ".join(toks)
		print("after",example["pointers"])
		print("after",example["sentence"])
		return None

		def decide(self,resp,i=0,nclasses=18,auxreg=True,softmax=True):
		"""map Likert scale to targets

		Args:
		resp: Likert class label
		i: property num
		auxreg: do we use auxiliary regression?
		softmax: is the main output multi label?

		Returns:
		normalized target as needed for the task at hand (regression or multi label)
		"""
		#if i is auxiliary output or main task is regression
		if (i > nclasses-1 and auxreg) or softmax==False:
		y = float(resp)
		return np.array([y])

		#set label to 1 if likert scale >= 4
		if float(resp) >= 4:
		return np.array([0,1])
		else:
		return np.array([1,0])

		def sortf(self,yss,catss,cats,nclasses,auxreg,softmax):
		"""sorts categories alphabetically and formats targets

		Args:
		yss: list of lists with labels e.g. [
		["1","2","5",1"]
		,...]
		Returns:
		ordered (according to alphabetically sorted categories) and formatted targets
		"""
		#2 times for main and different auxilliary
		cats = list(sorted(cats))*2
		ynews = []
		for cat in cats:
		ynews.append([])
		for i,cat in enumerate(cats):
		for j,ys in enumerate(yss):
		#print(catss[j])
		idx = catss[j].index(cat)
		resp = ys[idx]
		ynews[i].append(self.decide(resp,i=i,nclasses=nclasses,auxreg=auxreg,softmax=softmax))
		return ynews

		def retr(self,data,key="sentence",part="train"):
		"""retrieves instances from json file

		Args:
		data: data dict
		key: what to retrieve?
		part: train dev or test

		Returns:
		list with requested instances
		"""
		out = []

		keys=list(sorted(data.keys()))
		for k in keys:
		for example in data[k]:
		if example["split"] != part:
		continue
		else:
		out.append(example[key])
		return out

		def replacey(self,yss,ap,replacement=0):
		"""replaces targets in place as descirbed in prev work when the annotator answered with applicable=False

		Args:
		yss: list of lists with labels
		ap: list of lists with applicable labels
		replacement: what to use as replacement if not applicable

		Returns:
		None
		"""
		for i,ys in enumerate(yss):
		for j in range(len(ys)):
		if ap[i][j] == "False":
		ys[j] = replacement
		return None


		def avg_scores(self,example):
		"""if SPR2 we need to average the scores of two annotatorsi, inplace
		modification of the dict representing one data example

		Args:
		example: dictionary with "cat","label","applicable","annotator" keys
		which return corresponding lists
		Returns:
		None

		"""

		#sort according to annotators
		a = list(sorted(example["annotator"]))
		a1=a[0]
		a2=a[-1]
		a1i = [i for i,e in enumerate(example["annotator"]) if e == a1]
		a2i = [i for i,e in enumerate(example["annotator"]) if e == a2]
		example["cat"] = [example["cat"][i] for i in a1i] + [example["cat"][i] for i in a2i]
		example["label"] = [example["label"][i] for i in a1i] + [example["label"][i] for i in a2i]
		example["applicable"] = [example["applicable"][i] for i in a1i] + [example["applicable"][i] for i in a2i]
		example["annotator"] = [example["annotator"][i] for i in a1i] + [example["annotator"][i] for i in a2i]
		for i in range(len(example["applicable"])):
		if example["applicable"][i] == "False":
		example["applicable"][i] = "True"
		#example["label"][i] = "1"
		n=int(len(example["label"])/2)
		nbs=False
		for i,elm in enumerate(example["label"]):
		#print(example,"A")
		if i < n:
		example["label"][i] =str((float(example["label"][i])+float(example["label"][i+n]))/2)
		example["label"] = example["label"][:n]
		example["applicable"] = example["applicable"][:n]
		example["cat"] = example["cat"][:n]
		return None

		class DataSet():

		"""class for preparing the data

		"""


		def __init__(self,PATH_DATASET,LENGTH=30,AUXREG=True,SOFTMAX=True,PRETRAINED="bert-base-uncased"):
		self.PATH_DATASET = PATH_DATASET
		self.LENGTH = LENGTH
		self.AUXREG=AUXREG
		self.SOFTMAX = SOFTMAX
		self.PRETRAINED=PRETRAINED
		self.helpers = HelperFunctions()
		return None

		def prepare(self):
		"""prepares marker and word sequences and target labels

		Returns:
		None

		"""

		if "spr1" in self.PATH_DATASET:
		self.NCLASSES = 18
		if "spr2" in self.PATH_DATASET:
		self.NCLASSES = 14
		with open(self.PATH_DATASET,"r") as f:
		data = json.load(f)

		#make sure arg and pred are in length of seq:
		for key in data:
		for ex in data[key]:
		self.helpers.maybe_crop_and_adjust_input(ex,self.LENGTH)

		if "spr2" in self.PATH_DATASET:
		for key in data:
		data[key]=[elm for elm in data[key] if elm["ispilot"] == "False"]
		for example in data[key]:
		self.helpers.avg_scores(example)


		self.trainxwords = self.helpers.retr(data,key="sentence",part="train")#[:100]
		self.devxwords = self.helpers.retr(data,key="sentence",part="dev")#[:100]
		self.testxwords = self.helpers.retr(data,key="sentence",part="test")#[:100]



		self.trainxpointers = self.helpers.retr(data,key="pointers",part="train")#[:100]
		self.devxpointers = self.helpers.retr(data,key="pointers",part="dev")#[:100]
		self.testxpointers = self.helpers.retr(data,key="pointers",part="test")#[:100]


		trainy = self.helpers.retr(data,key="label",part="train")
		devy = self.helpers.retr(data,key="label",part="dev")
		testy = self.helpers.retr(data,key="label",part="test")
		if APPL2ZERO:
		traina = self.helpers.retr(data,key="applicable",part="train")
		deva = self.helpers.retr(data,key="applicable",part="dev")
		testa = self.helpers.retr(data,key="applicable",part="test")
		if SOFTMAX:
		# if main output is softmax we use 0 as auxiliary Liker target when rated inapplicable by annotator
		self.helpers.replacey(trainy,traina,"0")
		self.helpers.replacey(devy,deva,"0")
		self.helpers.replacey(testy,testa,"0")
		else:
		self.helpers.replacey(trainy,traina,"1")
		self.helpers.replacey(devy,deva,"1")
		self.helpers.replacey(testy,testa,"1")

		self.cats = sorted(list(self.helpers.retr(data,key="cat",part="train")[0]))#[data[key]["cat"] for key in keys if data[key]["split"] == "train"][0]))
		self.trainy=self.helpers.sortf(trainy,self.helpers.retr(data,key="cat",part="train"),self.cats,self.NCLASSES,self.AUXREG,self.SOFTMAX)
		self.devy=self.helpers.sortf(devy,self.helpers.retr(data,key="cat",part="dev"),self.cats, self.NCLASSES,self.AUXREG,self.SOFTMAX)
		self.testy=self.helpers.sortf(testy,self.helpers.retr(data,key="cat",part="test"),self.cats,self.NCLASSES,self.AUXREG,self.SOFTMAX)
		self.trainy = [np.array(y) for y in self.trainy]
		self.devy = [np.array(y) for y in self.devy]
		self.testy = [np.array(y) for y in self.testy]
		return None


		def _pointers2ints(self,ptrs):
		out=[]
		for string in ptrs:
		seq = [int(x) for x in string.split(" ")]
		while len(seq) < self.LENGTH:
		seq.insert(0,3)
		out.append(seq)
		return np.array(out)

		def make_data(self):
		"""generates matrices corresponding to train,dev and test instances and labels
		ready for feeding it to the nerual model

		Returns:
		trainxwords: n_exampleslength_seqdim_embedding matrix representing the training instances
		devxwords: same but dev
		testxwords: same but test
		trainxmarkers: n_examples*length_seq matrix consisting of integers which indicate whether
		a word is arg or pred or not
		devxmarkers: same but for dev
		testxmarkers: same but for test
		trainy: targets for train, normally list with 2n_properties numpy arrays, where first n_properties
		label are n_examples*one_hot_vec indicate multi-label targets and second n_properties
		are n_examples*float which indicate Likert variables
		devy: same but for dev
		testy: same but for test
		"""



		trainxpointers = self._pointers2ints(self.trainxpointers)
		devxpointers = self._pointers2ints(self.devxpointers)
		testxpointers = self._pointers2ints(self.testxpointers)
		print(devxpointers[:10])

		#embedding = BertEmbeddings(layers="-1")
		embedding = StackedEmbeddings([
		WordEmbeddings('glove'), BertEmbeddings(self.PRETRAINED,layers="-1,-2,-3,-4")])

		#embedding=BertEmbeddings(layers="-1")
		sent=Sentence("this is a test.")
		embedding.embed(sent)
		for word in sent:
		print(word.embedding.shape)


		def sumbert(x):
		"""sums the Bert embeddings over the last for layers, here first 100 dims represent glove
		Args:
		x: numpy 1d array with first 100 dims are glove dimensions and last 768*4 dims are bert dims
		Returns:
		1d array with reduced 100+768 dims because Bert features were summed over last for layers
		"""
		glo = x[:100]
		rest=x[100:]
		if "base" in BERT:
		rest=rest.reshape(4,768).sum(axis=0)
		else:
		rest=rest.reshape(4,1024).sum(axis=0)
		return np.concatenate((glo,rest))

		def zeropad(xss,dim=868):
		for xs in xss:
		while len(xs) < self.LENGTH:
		xs.insert(0,np.zeros(dim))
		return None

		def sum_and_save(tokenss,path):
		"""applies Bert layer sum over documents, saves final data matrices in file
		so that they don't need to be recomputed in every run

		Args:
		tokenss: list of list with flair token instances
		path: path where we save data matrices
		Returns:
		ready data matrices

		"""

		vecs=[[sumbert(t.embedding.numpy()) for t in tokens] for tokens in tokenss]
		zeropad(vecs)
		vecs=np.array(vecs)

		np.save(path,vecs)
		return vecs



		if "spr1" in self.PATH_DATASET:
		if os.path.isfile("bertbasetrain1.npy"):
		trainxwords=np.load("bertbasetrain1.npy")
		else:
		trainxwords = [Sentence(x,use_tokenizer=False) for x in self.trainxwords]
		print([len(x) for x in trainxwords])
		print("trainsents created")
		[embedding.embed(s) for s in trainxwords]
		print("trainvecs created")
		trainxwords = sum_and_save(trainxwords,"bertbasetrain1.npy")
		if os.path.isfile("bertbasedev1.npy"):
		devxwords=np.load("bertbasedev1.npy")
		else:
		devxwords = [Sentence(x,use_tokenizer=False) for x in self.devxwords]
		print([len(x) for x in devxwords])
		print("devsents created")
		[embedding.embed(s) for s in devxwords]
		print("devvecs created")
		devxwords=sum_and_save(devxwords,"bertbasedev1.npy")
		if os.path.isfile("bertbasetest1.npy"):
		testxwords=np.load("bertbasetest1.npy")
		else:
		testxwords = [Sentence(x,use_tokenizer=False) for x in self.testxwords]
		print([len(x) for x in testxwords])
		print("testsents created")
		[embedding.embed(s) for s in testxwords]
		print("testvecs created")
		testxwords=sum_and_save(testxwords,"bertbasetest1.npy")
		elif "spr2" in self.PATH_DATASET:
		if os.path.isfile("bertbasetrain2.npy"):
		trainxwords=np.load("bertbasetrain2.npy")
		else:
		trainxwords = [Sentence(x,use_tokenizer=False) for x in self.trainxwords]
		print([len(x) for x in trainxwords])
		print("trainsents created")
		[embedding.embed(s) for s in trainxwords]
		print("trainvecs created")
		trainxwords=sum_and_save(trainxwords,"bertbasetrain2.npy")
		if os.path.isfile("bertbasedev2.npy"):
		devxwords=np.load("bertbasedev2.npy")
		else:
		devxwords = [Sentence(x,use_tokenizer=False) for x in self.devxwords]
		print([len(x) for x in devxwords])
		print("devsents created")
		[embedding.embed(s) for s in devxwords]
		print("devvecs created")
		devxwords=sum_and_save(devxwords,"bertbasedev2.npy")
		if os.path.isfile("bertbasetest2.npy"):
		testxwords=np.load("bertbasetest2.npy")
		else:
		testxwords = [Sentence(x,use_tokenizer=False) for x in self.testxwords]
		print([len(x) for x in testxwords])
		print("testsents created")
		[embedding.embed(s) for s in testxwords]
		print("testvecs created")
		testxwords=sum_and_save(testxwords,"bertbasetest2.npy")

		print(trainxwords.shape)
		return trainxwords,devxwords,testxwords,trainxpointers,devxpointers,testxpointers,self.trainy,self.devy,self.testy