Upload New File (6efcd65e) · Commits · pirapakaran / Softwareproject_ws2020_21

Evaluation/train_evaluation.ipynb

0 → 100644

+379 −0

Original line number	Diff line number	Diff line
		%% Cell type:code id: tags:

		``` python
		#!/usr/bin/env python3
		# -- coding: utf-8 --
		"""
		Created on Tue Mar 23 02:03:59 2021

		@author: SWP-Group
		This file analyzes the training output and analyzes the classification of Reiss human needs
		on the training set.

		"""
		import pandas as pd

		#GS_annotations file should be in the same directory, else insert filepath
		output = pd.read_csv("knowlege_path_subset_train.csv", sep=';', error_bad_lines=False)
		index_list = output.index.tolist()
		#output["Essay"] += 1
		#print(index_list)
		essay_output_list = output["Essay"].tolist()
		essay_output_list = essay_output_list[:94]
		path_list = output["Path"].tolist()

		gold_df = pd.read_csv("input_subset_train.csv", sep=';', error_bad_lines=False)
		gold_essay_list = gold_df["Essay"].tolist()
		reiss_gold = gold_df["Reiss"].tolist()


		def replacer(list_string):
		"""
		Function to replace all the unnecessary characters in the paths in order
		to further process the data and return clean strings

		Parameters
		----------
		list_string : list
		a list containing strings of words (here: conceptnet paths
		as strings)

		Returns
		-------
		str
		strings cleaned from unwanted and unnecessary characters and tokens

		"""
		text = list_string.replace("[", "").replace("]", "").replace('\'', "").replace("\"", "").replace(",", "")
		return text.split()

		# create maslow and reiss human needs
		maslow_human_needs = ["physiological needs", "stability", "love/belonging", "esteem", "spiritual growth"]
		reiss_motives = ["food", "rest", "health", "save_money", "order", "safety",
		"romance", "belonging", "family", "contact", "competition",
		"honor", "approval", "status", "power", "curiosity", "serenity",
		"idealism", "independent"]


		# create a cleaned list of paths
		cleaned_paths = [replacer(path) for path in path_list]


		def assign_reiss(path_list):
		"""
		Assigns Reiss motive to a a graph consisting of a list of its paths

		Parameters
		----------
		path_list : list
		The entire list of subraphs which consist of their paths
		paths are split into their single units as strings

		Returns
		-------
		None.

		"""
		temp_list = []
		human_needs = []
		for path in path_list:
		for word in path:
		if word in reiss_motives:
		temp_list.append(word)
		human_needs.append(temp_list[0])
		temp_list = []
		return human_needs


		#assign reiss human needs for every graph via its top ranked path
		reiss_needs = assign_reiss(cleaned_paths)

		#maslow needs list to assign maslow need accordingly
		physiological_needs = ['food', 'rest']
		safety = ['health', 'save_money', 'order', 'safety']
		love_belonging = ['love', 'belonging', 'family', 'contact']
		esteem = ['competition','honor', 'approval', 'status', 'power']
		spiritual_growth = ['curiosity', 'serenity','idealism', 'independent']

		def assign_maslow(reiss_list):
		"""
		Function that assigns corresponding maslow human need given its
		reiss human need

		Parameters
		----------
		reiss_list : list
		list containing assigned reiss human need for every essay
		(via the top ranked graphpath)

		Returns
		-------
		maslow_needs : list
		list containing corresponding maslow human needs

		"""
		maslow_needs = []
		for r in reiss_needs:
		if r in physiological_needs:
		maslow_needs.append(maslow_human_needs[0])
		elif r in safety:
		maslow_needs.append(maslow_human_needs[1])
		elif r in love_belonging:
		maslow_needs.append(maslow_human_needs[2])
		elif r in esteem:
		maslow_needs.append(maslow_human_needs[3])
		else:
		maslow_needs.append(maslow_human_needs[4])
		return maslow_needs

		maslow_needs = assign_maslow(reiss_needs)

		# create joint list of reiss and maslow
		hn_list_full = list(zip(maslow_needs, reiss_needs))

		#post-processing for evaluation in reiss
		reiss_needs = [w.replace("independent", "independence") for w in reiss_needs]
		reiss_needs = [w.replace("save_money", "savings") for w in reiss_needs]

		#post-processing for evaluation in maslow
		maslow_needs = [w.replace("love / belonging", "love/belonging") for w in maslow_needs]

		# post-processing of gold data
		maslow_gold = [w.replace("love / belonging", "love/belonging") for w in maslow_gold]

		# add columns accordingly
		output["Maslow_predict"] = maslow_needs
		output["Reiss_predict"] = reiss_needs
		```

		%% Cell type:code id: tags:

		``` python
		output
		```

		%% Output

		Essay \
		0 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent1
		1 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent2
		2 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent3
		3 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent4
		4 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent5
		.. ...
		87 42b3c857-0370-4316-b42d-ae81d4bbe3bf__sent4
		88 42b3c857-0370-4316-b42d-ae81d4bbe3bf__sent5
		89 f89a1b4d-a8cc-4047-8844-b2862aaac0dd__sent3
		90 f89a1b4d-a8cc-4047-8844-b2862aaac0dd__sent4
		91 f89a1b4d-a8cc-4047-8844-b2862aaac0dd__sent5

		Path Maslow_predict \
		0 ['humiliate Antonym honor', 'honor RelatedTo t... esteem
		1 ['laugh MotivatedByGoal see_particular_program... spiritual growth
		2 ['challenge RelatedTo competition', 'competiti... esteem
		3 ['like IsA approval', 'approval RelatedTo look... esteem
		4 ['defeat RelatedTo competition', 'competition ... esteem
		.. ... ...
		87 ['stop RelatedTo rest', 'rest Antonym play', '... physiological needs
		88 ['play Antonym rest', 'rest RelatedTo evening ... physiological needs
		89 ['fight RelatedTo competition', 'competition R... esteem
		90 ['throw HasSubevent playing_frisbee UsedFor co... esteem
		91 ['buy Antonym save_money', 'save_money Desires... stability

		Reiss_predict
		0 honor
		1 curiosity
		2 competition
		3 approval
		4 competition
		.. ...
		87 rest
		88 rest
		89 competition
		90 competition
		91 savings

		[92 rows x 4 columns]

		%% Cell type:code id: tags:

		``` python
		gold_df
		```

		%% Output

		Essay Reiss
		0 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent1 competition
		1 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent2 status
		2 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent3 competition
		3 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent4 competition
		4 5acb6474-4a03-4f0c-9ef0-b59293b10384__sent5 power
		.. ... ...
		87 42b3c857-0370-4316-b42d-ae81d4bbe3bf__sent4 competition
		88 42b3c857-0370-4316-b42d-ae81d4bbe3bf__sent5 competition
		89 f89a1b4d-a8cc-4047-8844-b2862aaac0dd__sent3 status
		90 f89a1b4d-a8cc-4047-8844-b2862aaac0dd__sent4 order
		91 f89a1b4d-a8cc-4047-8844-b2862aaac0dd__sent5 family

		[92 rows x 2 columns]

		%% Cell type:code id: tags:

		``` python
		len(gold_essay_list)
		```

		%% Output

		92

		%% Cell type:markdown id: tags:

		# Evaluation of training output
		In the following sections, we will analyze the training output with the metrics provided by sklearn's "metric" module.

		%% Cell type:code id: tags:

		``` python
		from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
		from sklearn.metrics import ConfusionMatrixDisplay, classification_report, confusion_matrix
		```

		%% Cell type:markdown id: tags:

		### Accuracy

		%% Cell type:code id: tags:

		``` python
		train_acc = accuracy_score(reiss_gold, reiss_needs)
		print(train_acc)
		```

		%% Output

		0.14130434782608695

		%% Cell type:markdown id: tags:

		### Precision

		%% Cell type:code id: tags:

		``` python
		train_prec = precision_score(reiss_gold, reiss_needs, average='macro')
		print(train_prec)
		```

		%% Output

		0.08556480284421461

		%% Cell type:code id: tags:

		``` python
		train_recall = recall_score(reiss_gold, reiss_needs, average='macro')
		print(train_recall)
		```

		%% Output

		0.12695804195804197

		%% Cell type:markdown id: tags:

		### Recall

		%% Cell type:code id: tags:

		``` python
		train_f1score = f1_score(reiss_gold, reiss_needs, average="macro")
		print(train_f1score)
		```

		%% Output

		0.08443544759334232

		%% Cell type:markdown id: tags:

		### Classification Report

		%% Cell type:code id: tags:

		``` python
		class_report = classification_report(reiss_gold, reiss_needs)
		print(class_report)
		```

		%% Output

		precision recall f1-score support

		None 0.00 0.00 0.00 7
		approval 0.00 0.00 0.00 2
		belonging 0.00 0.00 0.00 1
		competition 0.38 0.38 0.38 13
		contact 0.00 0.00 0.00 10
		curiosity 0.14 0.20 0.17 5
		family 0.00 0.00 0.00 6
		food 0.62 0.45 0.53 11
		health 0.50 0.50 0.50 2
		honor 0.00 0.00 0.00 0
		idealism 0.00 0.00 0.00 1
		independence 0.00 0.00 0.00 0
		order 0.00 0.00 0.00 7
		power 0.00 0.00 0.00 2
		rest 0.06 1.00 0.11 1
		romance 0.00 0.00 0.00 7
		safety 0.00 0.00 0.00 0
		savings 0.00 0.00 0.00 2
		serenity 0.00 0.00 0.00 5
		status 0.00 0.00 0.00 10

		accuracy 0.14 92
		macro avg 0.09 0.13 0.08 92
		weighted avg 0.15 0.14 0.14 92


		%% Cell type:markdown id: tags:

		### Confusion matrix

		%% Cell type:code id: tags:

		``` python
		cm_reiss = confusion_matrix(reiss_gold, maslow_needs)
		print(cm_reiss)
		```

		%% Output

		[[0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0 0 4 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0]
		[0 0 0 0 0 0 7 0 0 0 0 0 0 3 0 0 0 0 0 0 3 0]
		[0 0 0 0 0 0 1 0 0 0 0 0 0 4 0 0 0 0 0 2 3 0]
		[0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 2 1 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
		[0 0 0 0 0 0 2 0 0 0 0 0 0 1 0 0 0 0 0 1 2 0]
		[0 0 0 0 0 0 1 0 0 0 0 1 0 6 0 0 0 0 0 2 1 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
		[0 0 0 0 0 0 2 0 0 0 0 0 0 1 0 0 0 0 0 2 2 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
		[0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0]
		[0 0 0 0 0 0 4 0 0 0 0 1 0 1 0 0 0 0 0 1 0 0]
		[0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 0 0 0]
		[0 0 0 0 0 0 1 0 0 0 0 1 0 2 0 0 0 0 0 0 1 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
		[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
		[0 0 0 0 0 0 2 0 0 0 0 0 0 3 0 0 0 0 0 4 1 0]]

		%% Cell type:code id: tags:

		``` python
		```