lstm works (bd38aed1) · Commits · Softwareprojekt 2018 - Jason Marinco Rudolf Samuel / O Caption My Caption

caption-lib/lstm/Models/coco_model.py

+44 −5

Original line number	Diff line number	Diff line
		@@ -15,7 +15,7 @@ from __future__ import division
		import tensorflow as tf

		class coco_model(object):
		def __init__(self, word_to_idx, dim_feature=[196, 512], dim_embed=512, dim_hidden=2048, n_time_step=16,
		def __init__(self, word_to_idx, curr_reader, dim_feature=[196, 512], dim_embed=512, dim_hidden=2048, n_time_step=16,
		prev2out=True, ctx2out=True, alpha_c=0.0, selector=True, dropout=True):
		"""
		Args:
		@@ -47,19 +47,36 @@ class coco_model(object):
		self._start = word_to_idx['<START>']
		self._null = word_to_idx['<NULL>']

		self.current_reader = curr_reader

		self.weight_initializer = tf.contrib.layers.xavier_initializer()
		self.const_initializer = tf.constant_initializer(0.0)
		self.emb_initializer = tf.random_uniform_initializer(minval=-1.0, maxval=1.0)

		# Place holder for features and captions
		self.features = tf.placeholder(tf.float32, [self.L, self.D], name="features_placeholder_model")
		self.captions = tf.placeholder(tf.int32, [self.T + 1], name="captions_placeholder_model")
		self.features = tf.placeholder(tf.float32, [None, self.D], name="features_placeholder_model")
		self.captions = tf.placeholder(tf.int32, [None, self.L], name="captions_placeholder_model")

		print("\n#>\t CREATE MODEL")
		print("#>\tdim_feature:", dim_feature)
		print("#>\tdim_embed", dim_embed)
		print("#>\tdim_hidden", dim_hidden)
		print("#>\tn_time_step", n_time_step)
		print("#>\tprev2out", prev2out)
		print("#>\tctx2out", ctx2out)
		print("#>\talpha_c", alpha_c)
		print("#>\tselector", selector)
		print("#>\tdropout", dropout)
		print("#>\tself.features", self.features)
		print("#>\tself.captions", self.captions)

		#self.features = tf.placeholder(tf.float32, [self.L, self.D], name="features_placeholder_model")
		#self.captions = tf.placeholder(tf.int32, [self.T + 1], name="captions_placeholder_model")

		def _get_initial_lstm(self, features):
		with tf.variable_scope('initial_lstm'):
		features_mean = tf.reduce_mean(features, 1)
		#features_mean = tf.reduce_mean(features, 1)
		features_mean = features

		w_h = tf.get_variable('w_h', [self.D, self.H], initializer=self.weight_initializer)
		b_h = tf.get_variable('b_h', [self.H], initializer=self.const_initializer)
		@@ -68,12 +85,26 @@ class coco_model(object):
		w_c = tf.get_variable('w_c', [self.D, self.H], initializer=self.weight_initializer)
		b_c = tf.get_variable('b_c', [self.H], initializer=self.const_initializer)
		c = tf.nn.tanh(tf.matmul(features_mean, w_c) + b_c)

		print("\n#>\t Initial LSTM")
		print("#>\t:", features_mean[0])
		print("#>\tc:", c)
		print("#>\th", h)

		return c, h

		def _word_embedding(self, inputs, reuse=False):
		with tf.variable_scope('word_embedding', reuse=reuse):
		w = tf.get_variable('w', [self.V, self.M], initializer=self.emb_initializer)
		x = tf.nn.embedding_lookup(w, inputs, name='word_vector') # (N, T, M) or (N, M)
		print("\n#>\t Embedding Loopkup")
		print("#>\tV:", self.V)
		print("#>\tM:", self.M)
		print("#>\tw: >", w)
		print("#>\tinputs:", inputs)
		print("#>\tx:", x)


		return x

		def _project_features(self, features):
		@@ -157,6 +188,12 @@ class coco_model(object):
		alpha_list = []
		lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=self.H)

		print("\n#>\t Build Model")
		print("#>\tmask:", mask)
		print("#>\tbatch_normalisation:", self._batch_norm)
		print("#>\tword_embedding:", x)
		#print("#>\tinputs:", inputs)

		for t in range(self.T):
		context, alpha = self._attention_layer(features, features_proj, h, reuse=(t!=0))
		alpha_list.append(alpha)
		@@ -169,7 +206,9 @@ class coco_model(object):

		logits = self._decode_lstm(x[:,t,:], h, context, dropout=self.dropout, reuse=(t!=0))

		loss += tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=captions_out[:, t],logits=logits)*mask[:, t] )
		sparse_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=captions_out[:, t],logits=logits)*mask[:, t]

		loss += tf.reduce_sum(sparse_cross_entropy)

		if self.alpha_c > 0:
		alphas = tf.transpose(tf.stack(alpha_list), (1, 0, 2)) # (N, T, L)

caption-lib/lstm/Readers/coco_reader.py

+90 −82

Original line number	Diff line number	Diff line
		import numpy as np
		from lstm.configuration import current as config
		from lstm.configuration import current as _config
		from lstm.Readers import helper
		import collections
		import pickle
		@@ -9,7 +9,7 @@ import json as jsonlib
		import re

		class coco_reader:
		def load_data(self, data_path=helper.inputPath, filename=config.path.input_train):
		def load_data(self, data_path=helper.inputPath, filename=_config.path.input_train):
		start_t = time.time()
		data = {}
		formatted_data = {}
		@@ -21,54 +21,92 @@ class coco_reader:
		with open(data_path + filename + '.pickle', 'rb') as handle:
		data = pickle.load(handle)

		"""
		# generate annotations
		for d in data:
		for x in d['captions']:
		formatted_data.append([x['caption'], d['file_name'], x['image_id']])
		"""
		for data_from_image in data:
		for i, data_from_image in enumerate(data):
		for image_captions_dict in data_from_image['captions']:
		#5 times
		#Image Append
		features.append(data_from_image['vector'])
		captions.append(data_from_image['captions'][0]['caption'])
		#captions.append(data_from_image['captions'][1]['caption'])
		#captions.append(data_from_image['captions'][2]['caption'])
		#captions.append(data_from_image['captions'][3]['caption'])
		#captions.append(data_from_image['captions'][4]['caption'])
		image_idxs.append(data_from_image['id'])

		if(config.reader.verbose):
		print('#> reader printing example data:')
		print('\t\tcaptions:', data_from_image['captions'][0]['caption'])
		#print('\t\tcaptions:', data[0]['captions'][1]['caption'])
		#print('\t\tcaptions:', data[0]['captions'][2]['caption'])
		#print('\t\tcaptions:', data[0]['captions'][3]['caption'])
		#print('\t\tcaptions:', data[0]['captions'][4]['caption'])
		print('\timage_id:', data_from_image['id'])
		print('\tvector:', data_from_image['vector'])
		#Caption Append
		startword = ['<START>']
		words = re.findall(r"\w+\|[^\w\s]", image_captions_dict['caption'])
		words.append('<END>')
		words = self.add_padding(startword + words)
		captions.append(np.array(words))
		#Image ID Append
		image_idxs.append(np.array(data_from_image['id']))

		if(_config.reader.verbose and i < 3):
		print('\n#> Reader printing ' + filename + ' data:')
		print('#>\tcaptions:', data_from_image['captions'][0]['caption'])
		print('\tcaptions:', data_from_image['captions'][1]['caption'])
		print('\tcaptions:', data_from_image['captions'][2]['caption'])
		print('\tcaptions:', data_from_image['captions'][3]['caption'])
		print('\tcaptions:', data_from_image['captions'][4]['caption'])
		print('#>\timage_id:', data_from_image['id'])
		print('#>\tvector:', data_from_image['vector'])

		#encode the captions
		captions = np.array(self.encode_captions(captions))

		formatted_data['captions'] = np.array(captions)
		formatted_data['image_idxs'] = np.array(image_idxs)
		formatted_data['features'] = np.array(features)

		if(_config.reader.verbose):
		print("#> Data shape:")
		print("#>\tfeature set count complete:", formatted_data['features'].shape)
		print("#>\tcaption set count complete:", formatted_data['captions'].shape)
		print("#>\timageId set count complete:", formatted_data['image_idxs'].shape)

		end_t = time.time()
		print("#> Load data elapsed time: %.2f" % (end_t - start_t))
		print("#>\tLoad data elapsed time: %.2f" % (end_t - start_t))
		return formatted_data
		"""
		def _build_indexed_array(self, masa_data, array):
		nd_array = np.ndarray(len(masa_data), dtype=np.int32)
		for i, value in enumerate(array):
		nd_array[i] = value
		return nd_array

		def _build_indexed_array_features(self, masa_data, array):
		nd_array = np.ndarray((len(masa_data), len(array)), dtype=np.int32)
		for i, value in enumerate(array):
		for x, t in enumerate(value):
		nd_array[i][x] = t
		return nd_array
		"""

		def load_data_masa(self, data_path=helper.inputPath, filename=config.path.input_train):

		def add_padding(self, sequence):
		new_seq = sequence
		seq_length = len(sequence)
		for i in range(_config.reader.sequence_length + 1):
		if i > seq_length:
		new_seq.append('<NULL>')

		return new_seq[:_config.reader.sequence_length]

		def decode_captions(self, encoded_captions):
		decoded = []
		vocab = self.get_stored_vocab_id_word(_config.path.input_train)

		for i, single_caption in enumerate(encoded_captions):
		decoded.append([])
		for word in single_caption:
		decoded[i].append(vocab[word])

		return np.array(decoded)

		def decode_captions_2(self, encoded_captions, id_to_word):
		decoded = []
		vocab = id_to_word

		for i, single_caption in enumerate(encoded_captions):
		decoded.append([])
		for word in single_caption:
		decoded[i].append(vocab[word])

		decoded = np.array(decoded)
		return decoded

		def encode_captions(self, decoded_captions):
		encoded = []
		vocab = self.get_stored_vocab_word_id(_config.path.input_train)

		for i, single_caption in enumerate(decoded_captions):
		encoded.append(np.arange(_config.reader.sequence_length))
		for t, word in enumerate(single_caption):
		encoded[i][t] = vocab[word]

		encoded = np.array(encoded)
		return encoded

		def load_data_masa(self, data_path=helper.inputPath, filename=_config.path.input_train):
		#this is the code to load our data like we planned
		#but since the model can be used differently we preprocess differently!
		start_t = time.time()
		@@ -78,14 +116,7 @@ class coco_reader:
		with open(data_path + filename + '.pickle', 'rb') as handle:
		data = pickle.load(handle)

		"""
		# generate annotations
		for d in data:
		for x in d['captions']:
		formatted_data.append([x['caption'], d['file_name'], x['image_id']])
		"""

		if(config.reader.verbose):
		if(False):
		for i in range(0, 3):
		print()
		print('#> read number: ', i)
		@@ -105,7 +136,7 @@ class coco_reader:
		print("Elapse time: %.2f" % (end_t - start_t))
		return data

		def create_vocab_from_filesystem(self, data_path: str=helper.inputPath, filename: str=config.path.input_train):
		def create_vocab_from_filesystem(self, data_path: str=helper.inputPath, filename: str=_config.path.input_train):
		data = self.load_data_masa(data_path, filename)
		return self._create_vocab_from_data(data, filename)

		@@ -119,7 +150,7 @@ class coco_reader:
		vocab_id_word[counter] = c
		counter = counter + 1
		if counter >= 39997:
		print('#> vocab exceeds over 40k abbort and add special characters')
		print('#>\tvocab exceeds over 40k abbort and add special characters')
		vocab_id_word[counter] = '<UNKN>'
		counter = counter + 1
		break
		@@ -129,7 +160,7 @@ class coco_reader:
		counter = counter + 1
		vocab_id_word[counter] = '<NULL>'
		counter = counter + 1
		print('#> final vocab size: ', len(vocab_id_word))
		print('#>\tfinal vocab size: ', len(vocab_id_word))

		vocab_word_id = {v: k for k, v in vocab_id_word.items()}

		@@ -139,45 +170,22 @@ class coco_reader:


		def get_stored_vocab_id_word(self, filename_prefix):
		with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_id_word' + config.reader.export_data_type, 'r') as outfile:
		with open(helper.parent_folder_path + _config.path.read_folder_path + filename_prefix + '_vocab_id_word' + _config.reader.export_data_type, 'r') as outfile:
		return jsonlib.load(outfile)

		def get_stored_vocab_word_id(self, filename_prefix):
		with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_word_id' + config.reader.export_data_type, 'r') as outfile:
		with open(helper.parent_folder_path + _config.path.read_folder_path + filename_prefix + '_vocab_word_id' + _config.reader.export_data_type, 'r') as outfile:
		return jsonlib.load(outfile)

		def save_vocab(self, filename_prefix, id_to_word, word_to_id):
		with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_id_word' + config.reader.export_data_type, 'w') as outfile:
		with open(helper.parent_folder_path + _config.path.read_folder_path + filename_prefix + '_vocab_id_word' + _config.reader.export_data_type, 'w') as outfile:
		jsonlib.dump(id_to_word, outfile)

		with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_word_id' + config.reader.export_data_type, 'w') as outfile:
		with open(helper.parent_folder_path + _config.path.read_folder_path + filename_prefix + '_vocab_word_id' + _config.reader.export_data_type, 'w') as outfile:
		jsonlib.dump(word_to_id, outfile)

		if(config.reader.verbose):
		print("Reader:: Saved vocabulary at:", filename_prefix)

		def decode_captions(self, captions, vocabulary_idx_to_word):
		if captions.ndim == 1:
		T = captions.shape[0]
		N = 1
		else:
		N, T = captions.shape

		decoded = []
		for i in range(N):
		words = []
		for t in range(T):
		if captions.ndim == 1:
		word = vocabulary_idx_to_word[captions[t]]
		else:
		word = vocabulary_idx_to_word[captions[i, t]]
		if word == '<END>':
		words.append('.')
		break
		if word != '<NULL>':
		words.append(word)
		decoded.append(' '.join(words))
		return decoded
		if(_config.reader.verbose):
		print("\n#>\t Saved vocabulary at:", filename_prefix)

		def sample_coco_minibatch(self, data, batch_size):
		data_size = data['features'].shape[0]

caption-lib/lstm/Trainers/coco_trainer.py

+88 −33

Original line number	Diff line number	Diff line
		@@ -6,6 +6,7 @@ import time
		import os
		from scipy import ndimage
		from lstm.configuration import current as _config
		import lstm.Readers as _reader

		class coco_trainer(object):
		def __init__(self, model, data, val_data, **kwargs):
		@@ -59,18 +60,24 @@ class coco_trainer(object):
		if not os.path.exists(self.log_path):
		os.makedirs(self.log_path)

		def train(self):
		print("\n#>\t INIT TRAINING")
		print("#>\ttf.Optimizer", self.optimizer)
		print("#>\tlearningrate", self.learning_rate)
		print("#>\tupdate_rule", self.update_rule)


		def train(self, current_reader):
		# train/val dataset
		# Changed this because I keep less features than captions, see prepro
		# n_examples = self.data['captions'].shape[0]

		n_examples = self.data['features'].shape[0]
		n_iters_per_epoch = int(np.ceil(float(n_examples) / self.batch_size))
		n_iters_per_epoch = int(np.floor(float(n_examples) / self.batch_size))
		features = self.data['features']
		captions = self.data['captions']
		image_idxs = self.data['image_idxs']
		val_features = self.val_data['features']
		n_iters_val = int(np.ceil(float(val_features.shape[0]) / self.batch_size))
		n_iters_val = int(np.floor(float(val_features.shape[0]) / self.batch_size))

		# build graphs for training model and sampling captions
		# This scope fixed things!!
		@@ -78,6 +85,9 @@ class coco_trainer(object):
		loss = self.model.build_model()
		tf.get_variable_scope().reuse_variables()
		_, _, generated_captions = self.model.build_sampler(max_len=20)
		print("\n#>\t INIT LOSS")
		print("#>\tloss", loss)


		# train op
		with tf.variable_scope(tf.get_variable_scope(), reuse=False):
		@@ -85,6 +95,11 @@ class coco_trainer(object):
		grads = tf.gradients(loss, tf.trainable_variables())
		grads_and_vars = list(zip(grads, tf.trainable_variables()))
		train_op = optimizer.apply_gradients(grads_and_vars=grads_and_vars)
		print("\n#>\t TRAIN Operation")
		print("#>\toptimizer\n", optimizer)
		# print("#>\tgradients\n", grads)
		# print("#>\tgrads_and_vars", grads_and_vars)
		# print("#>\train_op", train_op)

		# summary op
		# tf.scalar_summary('batch_loss', loss)
		@@ -99,10 +114,11 @@ class coco_trainer(object):
		#summary_op = tf.merge_all_summaries()
		summary_op = tf.summary.merge_all()

		print("The number of epoch: %d" %self.n_epochs)
		print("Data size: %d" %n_examples)
		print("Batch size: %d" %self.batch_size)
		print("Iterations per epoch: %d" %n_iters_per_epoch)
		print("\n#>\t TRAINING")
		print("#>\tTotal number of epochs: %d" %self.n_epochs)
		print("#>\tData size: %d" %n_examples)
		print("#>\tBatch size: %d" %self.batch_size)
		print("#>\tIterations per epoch: %d" %n_iters_per_epoch)

		config = tf.ConfigProto(allow_soft_placement = True)
		#config.gpu_options.per_process_gpu_memory_fraction=0.9
		@@ -121,35 +137,32 @@ class coco_trainer(object):
		curr_loss = 0
		start_t = time.time()

		print("#>\tfeature shape:", features.shape)
		print("#>\tcaption shape:", captions.shape)
		print("#>\timageId shape:", image_idxs.shape)
		for e in range(self.n_epochs):
		print("#>\tEPOCH")

		rand_idxs = np.random.permutation(n_examples)
		m_captions = captions[rand_idxs]
		m_image_idxs = image_idxs[rand_idxs]
		m_features = features[rand_idxs]

		for i in range(n_iters_per_epoch):

		captions_batch = m_captions[iself.batch_size:(i+1)self.batch_size]
		image_idxs_batch = m_image_idxs[iself.batch_size:(i+1)self.batch_size]
		features_batch = m_features[iself.batch_size:(i+1)self.batch_size]
		if i % 10 == 0:
		print("#> Batch Shapes")
		print("#>\tBatch: " + str(i) + "\tEpoch: " + str(e) + "\tBatch size: " + str(iself.batch_size) + " - " + str((i+1)self.batch_size))
		print("#>\tfrom Feature Batch:", features_batch.shape, features_batch[0].shape)
		print("#>\tfrom Captions Batch:", captions_batch.shape)

		print()
		print()
		print()
		print()
		print()
		print()
		print("image", features_batch[0])
		print("image_idx", image_idxs_batch[0])
		print("caption", captions_batch[0])

		#Here was an ERROR
		feed_dict = {self.model.features: features_batch, self.model.captions: captions_batch}
		print()
		print()
		print("Feed Dict:")
		print(feed_dict)
		print()
		print()

		_, l = sess.run([train_op, loss], feed_dict)
		curr_loss += l

		@@ -159,14 +172,16 @@ class coco_trainer(object):
		summary_writer.add_summary(summary, e*n_iters_per_epoch + i)

		if (i+1) % self.print_every == 0:
		print("\nTrain loss at epoch %d & iteration %d (mini-batch): %.5f" %(e+1, i+1, l))
		ground_truths = m_captions[image_idxs == image_idxs_batch[0]]
		decoded = decode_captions(ground_truths, self.model.idx_to_word)
		print("\nTrain loss at epoch %d & iteration %d (mini-batch): %.5f" %(e, i, l))
		print("#>\tImage:", image_idxs_batch[0])
		ground_truths = m_captions[m_image_idxs == image_idxs_batch[0]]
		decoded = current_reader.decode_captions_2(ground_truths, self.model.idx_to_word)
		for j, gt in enumerate(decoded):
		print("Ground truth %d: %s" %(j+1, gt))
		print("#>\tGround truth %d: %s" % (j, gt))
		gen_caps = sess.run(generated_captions, feed_dict)
		decoded = decode_captions(gen_caps, self.model.idx_to_word)
		print("Generated caption: %s\n" %decoded[0])
		decoded = current_reader.decode_captions_2(gen_caps, self.model.idx_to_word)
		print("#>\tGenerated caption: %s\n" % decoded[0])
		print("#>\tloss: ", curr_loss)

		print("Previous epoch loss: ", prev_loss)
		print("Current epoch loss: ", curr_loss)
		@@ -176,6 +191,7 @@ class coco_trainer(object):

		# print out BLEU scores and file write
		if self.print_bleu:
		#TODO this does not work!
		all_gen_cap = np.ndarray((val_features.shape[0], 20))
		for i in range(n_iters_val):
		features_batch = val_features[iself.batch_size:(i+1)self.batch_size]
		@@ -183,15 +199,54 @@ class coco_trainer(object):
		gen_cap = sess.run(generated_captions, feed_dict=feed_dict)
		all_gen_cap[iself.batch_size:(i+1)self.batch_size] = gen_cap

		all_decoded = decode_captions(all_gen_cap, self.model.idx_to_word)
		save_pickle(all_decoded, "./data/val/val.candidate.captions.pkl")
		all_decoded = current_reader.decode_captions_2(all_gen_cap, self.model.idx_to_word)
		self.save_pickle(all_decoded, "./data/val/val.candidate.captions.pkl")
		scores = evaluate(data_path='./data', split='val', get_scores=True)
		write_bleu(scores=scores, path=self.model_path, epoch=e)
		self.write_bleu(scores=scores, path=self.model_path, epoch=e)

		# save model's parameters
		if (e+1) % self.save_every == 0:
		if (e) % self.save_every == 0:
		saver.save(sess, os.path.join(self.model_path, 'model'), global_step=e+1)
		print("model-%s saved." %(e+1))
		print("#>\t SAVER SAVE")
		print("#>\tmodel-%s saved." % (e + 1))


		def sample_coco_minibatch(data, batch_size):
		#TODO rework
		data_size = data['features'].shape[0]
		mask = np.random.choice(data_size, batch_size)
		features = data['features'][mask]
		file_names = data['file_names'][mask]
		return features, file_names

		def write_bleu(scores, path, epoch):
		#TODO rework
		if epoch == 0:
		file_mode = 'w'
		else:
		file_mode = 'a'
		with open(os.path.join(path, 'val.bleu.scores.txt'), file_mode) as f:
		f.write('Epoch %d\n' %(epoch+1))
		f.write('Bleu_1: %f\n' %scores['Bleu_1'])
		f.write('Bleu_2: %f\n' %scores['Bleu_2'])
		f.write('Bleu_3: %f\n' %scores['Bleu_3'])
		f.write('Bleu_4: %f\n' %scores['Bleu_4'])
		f.write('METEOR: %f\n' %scores['METEOR'])
		f.write('ROUGE_L: %f\n' %scores['ROUGE_L'])
		f.write('CIDEr: %f\n\n' %scores['CIDEr'])

		def load_pickle(path):
		#TODO rework
		with open(path, 'rb') as f:
		file = pickle.load(f)
		print ('Loaded %s..' %path)
		return file

		def save_pickle(data, path):
		#TODO rework
		with open(path, 'wb') as f:
		pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
		print ('Saved %s..' %path)

		def test(self, data, split='train', attention_visualization=True, save_sampled_captions=True):
		'''

caption-lib/lstm/configuration.py

+3 −2

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ class default_reader(object): # default reader config
		verbose = False
		read_raw_files = False # this will only read files use only for testing
		create_vocab = True
		sequence_length = 20
		# current_reader = "basic_text_reader.py"

		class default_validator(object):
		@@ -83,10 +84,10 @@ class current(default_reader, default_validator, default_trainer, default_paths)
		# e.g. validator.read_test = "/test1234"

		reader.reader_on = False
		reader.verbose = False
		reader.verbose = True

		validator.validator_on = False

		trainer.train_on = True
		trainer.verbose = False
		trainer.verbose = True
		trainer.num_epochs = 3

caption-lib/lstm/lstm_main.py

+10 −8

File changed.

Preview size limit exceeded, changes collapsed.

Admin message