Commit 9e613e34 authored by Rudolf Chrispens's avatar Rudolf Chrispens
Browse files

added new model and ne reader and and and and...

parent 4dadd1de

caption-lib/log/events.out.tfevents.1529954234.Rudolfs-MacBook-Pro.local

0 → 100644
+2.49 MiB

File added.

No diff preview for this file type.

View file

caption-lib/log/events.out.tfevents.1529954349.Rudolfs-MacBook-Pro.local

0 → 100644
+2.51 MiB

File added.

No diff preview for this file type.

View file

caption-lib/lstm/Models/coco_model.py

+0 −1
Original line number Diff line number Diff line
@@ -144,7 +144,6 @@ class coco_model(object): 
        captions_out = captions[:, 1:]

        mask = tf.to_float(tf.not_equal(captions_out, self._null))





        # batch normalize feature vectors

        features = self._batch_norm(features, mode='train', name='conv_features')

caption-lib/lstm/Readers/coco_reader.py

+55 −8
Original line number Diff line number Diff line
@@ -10,6 +10,52 @@ import re 


class coco_reader:

    def load_data(self, data_path=helper.inputPath, filename=config.path.input_train):

        start_t = time.time()

        data = {}

        formatted_data = {}



        features = []

        captions = []

        image_idxs = []



        with open(data_path + filename + '.pickle', 'rb') as handle:

            data = pickle.load(handle)



        """

        # generate annotations

        for d in data:

            for x in d['captions']:

                formatted_data.append([x['caption'], d['file_name'], x['image_id']])

        """



        for data_from_image in data:

            features.append(data_from_image['vector'])

            captions.append(data_from_image['captions'][0]['caption'])

            captions.append(data_from_image['captions'][1]['caption'])

            captions.append(data_from_image['captions'][2]['caption'])

            captions.append(data_from_image['captions'][3]['caption'])

            captions.append(data_from_image['captions'][4]['caption'])

            image_idxs.append(data_from_image['id'])



            if(config.reader.verbose):

                print('#> reader printing example data:')

                print('\t\tcaptions:', data[0]['captions'][0]['caption'])

                print('\t\tcaptions:', data[0]['captions'][1]['caption'])

                print('\t\tcaptions:', data[0]['captions'][2]['caption'])

                print('\t\tcaptions:', data[0]['captions'][3]['caption'])

                print('\t\tcaptions:', data[0]['captions'][4]['caption'])

                print('\tvector:', data[0]['vector'])

        formatted_data['features'] = np.array(features)

        formatted_data['captions'] = np.array(captions)

        formatted_data['image_idxs'] = np.array(image_idxs)



        end_t = time.time()

        print("#> Load data elapsed time: %.2f" % (end_t - start_t))

        return formatted_data



    def load_data_masa(self, data_path=helper.inputPath, filename=config.path.input_train):

        #this is the code to load our data like we planned

        #but since the model can be used differently we preprocess differently!

        start_t = time.time()

        data = {}

        # formatted_data = []
@@ -45,10 +91,10 @@ class coco_reader: 
        return data



    def create_vocab_from_filesystem(self, data_path: str=helper.inputPath, filename: str=config.path.input_train):

        data = self.load_data(data_path, filename)

        return self.create_vocab_from_data(data)

        data = self.load_data_masa(data_path, filename)

        return self._create_vocab_from_data(data, filename)



    def create_vocab_from_data(self, data, filename):

    def _create_vocab_from_data(self, data, save_filename_prefix):

        counter: int = 0

        vocab_id_word = {}

        for x in data:
@@ -72,10 +118,11 @@ class coco_reader: 


        vocab_word_id = {v: k for k, v in vocab_id_word.items()}



        self.save_vocab(filename, vocab_id_word, vocab_word_id)

        self.save_vocab(save_filename_prefix, vocab_id_word, vocab_word_id)



        return vocab_word_id





    def get_stored_vocab_id_word(self, filename_prefix):

        with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_id_word' + config.reader.export_data_type, 'r') as outfile:

            return jsonlib.load(outfile)
@@ -84,15 +131,15 @@ class coco_reader: 
        with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_word_id' + config.reader.export_data_type, 'r') as outfile:

            return jsonlib.load(outfile)



    def save_vocab(self, filename, id_to_word, word_to_id):

        with open(helper.parent_folder_path + config.path.read_folder_path + filename + '_vocab_id_word' + config.reader.export_data_type, 'w') as outfile:

    def save_vocab(self, filename_prefix, id_to_word, word_to_id):

        with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_id_word' + config.reader.export_data_type, 'w') as outfile:

            jsonlib.dump(id_to_word, outfile)



        with open(helper.parent_folder_path + config.path.read_folder_path + filename + '_vocab_word_id' + config.reader.export_data_type, 'w') as outfile:

        with open(helper.parent_folder_path + config.path.read_folder_path + filename_prefix + '_vocab_word_id' + config.reader.export_data_type, 'w') as outfile:

            jsonlib.dump(word_to_id, outfile)



        if(config.reader.verbose):

            print("Reader:: Saved vocabulary at:", filename)

            print("Reader:: Saved vocabulary at:", filename_prefix)



    def decode_captions(self, captions, vocabulary_idx_to_word):

        if captions.ndim == 1:

caption-lib/lstm/Trainers/coco_trainer.py

+4 −4
Original line number Diff line number Diff line
@@ -4,10 +4,8 @@ import skimage.transform 
import numpy as np

import time

import os

import pickle

from scipy import ndimage

from utils import *

from bleu import evaluate

from lstm.configuration import current as _config



class coco_trainer(object):

    def __init__(self, model, data, val_data, **kwargs):
@@ -65,6 +63,7 @@ class coco_trainer(object): 
        # train/val dataset

        # Changed this because I keep less features than captions, see prepro

        # n_examples = self.data['captions'].shape[0]



        n_examples = self.data['features'].shape[0]

        n_iters_per_epoch = int(np.ceil(float(n_examples) / self.batch_size))

        features = self.data['features']
@@ -112,7 +111,7 @@ class coco_trainer(object): 
            tf.global_variables_initializer().run()

            #summary_writer = tf.train.SummaryWriter(self.log_path, graph=tf.get_default_graph())

            summary_writer = tf.summary.FileWriter(self.log_path, graph=tf.get_default_graph())

            saver = tf.train.Saver(max_to_keep=40)

            saver = tf.train.Saver(max_to_keep=10)



            if self.pretrained_model is not None:

                print("Start training with pretrained Model..")
@@ -131,6 +130,7 @@ class coco_trainer(object): 
                    captions_batch = captions[i*self.batch_size:(i+1)*self.batch_size]

                    image_idxs_batch = image_idxs[i*self.batch_size:(i+1)*self.batch_size]

                    features_batch = features[image_idxs_batch]

                    #Here was an ERROR

                    feed_dict = {self.model.features: features_batch, self.model.captions: captions_batch}

                    _, l = sess.run([train_op, loss], feed_dict)

                    curr_loss += l