Demo attention now properly padded still attention not working... (408b0aa1) · Commits · Softwareprojekt 2018 - Jason Marinco Rudolf Samuel / O Caption My Caption

caption_lib/LSTM.ipynb

+26 −40

File changed.

Preview size limit exceeded, changes collapsed.

caption_lib/lstm/Models/coco_model.py

+2 −1

Original line number	Diff line number	Diff line
		@@ -264,7 +264,8 @@ class coco_model(object):
		return alphas, betas, sampled_captions

		def build_sampler_test(self, features_to_test, max_len=20):
		features = self.features
		#features = self.features
		features = features_to_test

		# batch normalize feature vectors
		features = self._batch_norm(features, mode='test', name='conv_features')

caption_lib/lstm/Validators/coco_validator.py

+124 −9

Original line number	Diff line number	Diff line
		@@ -55,6 +55,13 @@ class coco_validator(object):
		- save_sampled_captions: If True, save sampled captions to pkl file for computing BLEU scores.
		'''

		if(data['features'].size < self.batch_size):
		# add fictional padding to feed it right
		for i in range(0, self.batch_size):
		for key, value in data.iteritems():
		value.append(value[0])

		# print("FINISHED!!!!!!! data!: ", data['features'].shape)
		features = data['features']
		image_idxs = data['image_idxs']
		# _config.path.image_data_path
		@@ -66,7 +73,7 @@ class coco_validator(object):
		config = tf.ConfigProto(allow_soft_placement=True)
		config.gpu_options.allow_growth = True
		with tf.Session(config=config) as sess:
		print("#>\tValidation session run: " + split + " " + str(features.shape))
		# print("#>\tValidation session run: " + split + " " + str(features.shape))
		saver = tf.train.Saver()
		saver.restore(sess, self.test_model)

		@@ -75,7 +82,9 @@ class coco_validator(object):
		image_files = sample_image_names

		feed_dict = {self.model.features: features_batch}

		alps, bts, sam_cap = sess.run([alphas, betas, sampled_captions], feed_dict) # (N, max_len, L), (N, max_len)

		# decoded = decode_captions(sam_cap, self.model.idx_to_word)
		decoded = current_reader.decode_captions_2(sam_cap, self.model.idx_to_word)

		@@ -90,12 +99,10 @@ class coco_validator(object):

		if _config.validator.show_attention:
		for n in range(_config.validator.attention_sample_number):
		print("#>\tSampled Caption: %s" % ' '.join(decoded[n]))


		#print("#>\tSampled Caption: %s" % ' '.join(decoded[n]))

		# Plot original image
		print("#>\tPlot attention image:" + image_path + "/" + str(image_files[n]))
		# print("#>\tPlot attention image:" + image_path + "/" + str(image_files[n]))
		img = ndimage.imread(image_path + "/" + str(image_files[n]))

		#image big:
		@@ -140,10 +147,7 @@ class coco_validator(object):
		plt.show()
		plt.close()




		if save_sampled_captions:
		if _config.validator.save_sampled_captions is True:
		num_iter = int(np.ceil(features.shape[0] / self.batch_size)) - self.batch_size
		print(features.shape)
		all_sam_cap = np.ndarray((features.shape[0], 20))
		@@ -183,4 +187,115 @@ class coco_validator(object):
		self.save_pickle(captionlist_for_eval, path + split + ".candidate.captions.pickle")
		self.save_json(captionlist_for_eval, path + split + ".candidate.captions.json")

		def test_demo(self, data, test_data, current_reader, split='train', attention_visualization=True, save_sampled_captions=True):
		'''
		Args:
		- data: dictionary with the following keys:
		- features: Feature vectors of shape (5000, 196, 512)
		- file_names: Image file names of shape (5000, )
		- captions: Captions of shape (24210, 17)
		- image_idxs: Indices for mapping caption to image of shape (24210, )
		- features_to_captions: Mapping feature to captions (5000, 4~5)
		- split: 'train', 'val' or 'test'
		- attention_visualization: If True, visualize attention weights with images for each sampled word. (ipthon notebook)
		- save_sampled_captions: If True, save sampled captions to pkl file for computing BLEU scores.
		'''

		if(test_data['features'].shape[0] < self.batch_size):
		# add fictional padding to feed it right
		for key, value in test_data.items():
		if key is 'captions' or key is 'features':
		test = np.vstack((value, value))
		value = np.vstack((test, test))
		test_data[key] = value
		else:
		value = np.repeat(value, 4)
		test_data[key] = value

		print("FINAL: ", test_data['features'].shape[0])

		# print("FINISHED!!!!!!! data!: ", data['features'].shape)
		features = test_data['features']#data['features']
		# image_idxs = data['image_idxs']
		# _config.path.image_data_path


		# build a graph to sample captions
		alphas, betas, sampled_captions = self.model.build_sampler_test(features_to_test=features, max_len=20) # (N, max_len, L), (N, max_len)

		config = tf.ConfigProto(allow_soft_placement=True)
		config.gpu_options.allow_growth = True
		with tf.Session(config=config) as sess:
		# print("#>\tValidation session run: " + split + " " + str(features.shape))
		saver = tf.train.Saver()
		saver.restore(sess, self.test_model)

		features_batch, sample_image_names, sample_captions = self.sample_coco_minibatch(test_data, self.batch_size)

		image_files = sample_image_names

		feed_dict = {self.model.features: features_batch}

		alps, bts, sam_cap = sess.run([alphas, betas, sampled_captions], feed_dict) # (N, max_len, L), (N, max_len)

		# decoded = decode_captions(sam_cap, self.model.idx_to_word)
		decoded = current_reader.decode_captions_2(sam_cap, self.model.idx_to_word)

		image_path = _config.path.image_data_path + _config.path.image_validation
		if split is 'train':
		image_path = _config.path.image_data_path + _config.path.image_train
		elif split is 'test':
		image_path = _config.path.image_data_path + _config.path.image_test
		else:
		image_path = _config.path.image_data_path + _config.path.image_validation


		if _config.validator.show_attention:
		for n in range(_config.validator.attention_sample_number):
		#print("#>\tSampled Caption: %s" % ' '.join(decoded[n]))

		# Plot original image
		# print("#>\tPlot attention image:" + image_path + "/" + str(image_files[n]))
		img = ndimage.imread(image_path + "/" + str(image_files[n]))

		#image big:
		plt.imshow(img)
		plt.axis('off')
		plt.show()
		plt.close()

		plt.subplot(4, 5, 1)

		plt.imshow(img)
		plt.axis('off')

		# Plot images with attention weights
		words = []
		for word in decoded[n]:
		words.append(word)
		#words = decoded[n].split(" ")
		for t in range(len(words)):
		if t > 11:
		break

		# IMAGE
		plt.subplot(4, 5, t + 2)
		img_resized = skimage.transform.resize(img, (299, 299), mode='reflect')
		plt.imshow(img_resized, cmap='gray', alpha=1)
		#plt.axis('off')
		# ATTENTION
		#plt.subplot(4, 5, t + 2)
		plt.text(0, 1, '%s' % words[t], color='black', backgroundcolor='white', fontsize=8)
		alp_curr = alps[n, t, :].reshape(4, 5)
		alp_img = skimage.transform.pyramid_expand(alp_curr, upscale=16, sigma=20)
		alp_img = skimage.transform.resize(alp_img, (299, 299), mode='reflect')
		plt.imshow(alp_img, alpha=0.5)
		plt.axis('off')

		if words[t] == '<END>':
		break


		plt.figure(figsize=(12, 12))
		plt.show()
		plt.close()

caption_lib/lstm/configuration.py

+19 −3

Original line number	Diff line number	Diff line
		@@ -46,6 +46,7 @@ class default_validator(object):
		attention_sample_number = 1
		show_attention = False
		max_iteration = -1
		save_sampled_captions = True

		class default_trainer(object):
		train_on = False
		@@ -76,18 +77,33 @@ class current(default_reader, default_validator, default_trainer, default_paths)
		trainer = default_trainer
		validator = default_validator

		print("#> Config: USING DEMO SETTINGS!")
		path.demo_sample_on = True
		# setting path to data that gets used by our trained model
		path.demo_sample_path = "/../../../Demo/example"

		# setting path to image file for attention visualisation
		path.image_data_path = os.path.dirname(__file__) + "/../../Demo/used_images_for_demo_notebook"
		path.image_validation = ''

		# enable validation and attention
		validator.max_iteration = 1
		validator.validator_on = True
		validator.show_attention = True
		validator.save_sampled_captions = False

		def demo(self):
		print("#> Config: USING DEMO SETTINGS!")
		self.path.demo_sample_on = True
		# setting path to data that gets used by our trained model
		self.path.demo_sample_path = "/../../../Demo/example.pickle"
		self.path.demo_sample_path = "/../../../Demo/example"

		# setting path to image file for attention visualisation
		self.path.image_data_path = os.path.dirname(__file__) + "/../../Demo/used_images_for_demo_notebook"
		self.path.image_validation = '/val'
		self.path.image_validation = ''

		# enable validation and attention
		self.validator.max_iteration = 1
		self.validator.validator_on = True
		self.validator.show_attention = True

		self.validator.save_sampled_captions = False

caption_lib/lstm/lstm_main.py

+3 −2

Original line number	Diff line number	Diff line
		@@ -84,8 +84,9 @@ def main(self, parameter_list):

		# change sample
		if _config.path.demo_sample_on:
		val_data = current_reader.load_data_encoded(filename=_config.path.demo_sample_path)

		val_test = current_reader.load_data_encoded(filename=_config.path.demo_sample_path)
		validator.test_demo(train_data, val_test, current_reader=current_reader, split='val', attention_visualization=True, save_sampled_captions=True)
		else:
		# validate trained
		validator.test(val_data, current_reader=current_reader, split='val', attention_visualization=True, save_sampled_captions=True)

Admin message