new ms coco reader validator trainer inserted but not working

capiton-lib/lstm/Trained_Data/*

caption-lib/.vscode/settings.json

.vscode/settings.json

*.pickle

# Created by https://www.gitignore.io/api/linux,macos,python,sublimetext,jupyternotebook,visualstudiocode

# =========================================================================================

# Implementation of "Show, Attend and Tell: Neural Caption Generator With Visual Attention".

# There are some notations.

# N is batch size.

# L is spacial size of feature vector (196).

# D is dimension of image feature vector (512).

# T is the number of time step which is equal to caption's length-1 (16).

# V is vocabulary size (about 10000).

# M is dimension of word vector which is embedding size (default is 512).

# H is dimension of hidden state (default is 1024).

# =========================================================================================

from __future__ import division

import tensorflow as tf

class CaptionGenerator(object):

    def __init__(self, word_to_idx, dim_feature=[196, 512], dim_embed=512, dim_hidden=1024, n_time_step=16,

                  prev2out=True, ctx2out=True, alpha_c=0.0, selector=True, dropout=True):

        """

        Args:

            word_to_idx: word-to-index mapping dictionary.

            dim_feature: (optional) Dimension of vggnet19 conv5_3 feature vectors.

            dim_embed: (optional) Dimension of word embedding.

            dim_hidden: (optional) Dimension of all hidden state.

            n_time_step: (optional) Time step size of LSTM.

            prev2out: (optional) previously generated word to hidden state. (see Eq (7) for explanation)

            ctx2out: (optional) context to hidden state (see Eq (7) for explanation)

            alpha_c: (optional) Doubly stochastic regularization coefficient. (see Section (4.2.1) for explanation)

            selector: (optional) gating scalar for context vector. (see Section (4.2.1) for explanation)

            dropout: (optional) If true then dropout layer is added.

        """

        self.word_to_idx = word_to_idx

        self.idx_to_word = {i: w for w, i in word_to_idx.iteritems()}

        self.prev2out = prev2out

        self.ctx2out = ctx2out

        self.alpha_c = alpha_c

        self.selector = selector

        self.dropout = dropout

        self.V = len(word_to_idx)

        self.L = dim_feature[0]

        self.D = dim_feature[1]

        self.M = dim_embed

        self.H = dim_hidden

        self.T = n_time_step

        self._start = word_to_idx['<START>']

        self._null = word_to_idx['<NULL>']

        self.weight_initializer = tf.contrib.layers.xavier_initializer()

        self.const_initializer = tf.constant_initializer(0.0)

        self.emb_initializer = tf.random_uniform_initializer(minval=-1.0, maxval=1.0)

        # Place holder for features and captions

        self.features = tf.placeholder(tf.float32, [None, self.L, self.D])

        self.captions = tf.placeholder(tf.int32, [None, self.T + 1])

    def _get_initial_lstm(self, features):

        with tf.variable_scope('initial_lstm'):

            features_mean = tf.reduce_mean(features, 1)

            w_h = tf.get_variable('w_h', [self.D, self.H], initializer=self.weight_initializer)

            b_h = tf.get_variable('b_h', [self.H], initializer=self.const_initializer)

            h = tf.nn.tanh(tf.matmul(features_mean, w_h) + b_h)

            w_c = tf.get_variable('w_c', [self.D, self.H], initializer=self.weight_initializer)

            b_c = tf.get_variable('b_c', [self.H], initializer=self.const_initializer)

            c = tf.nn.tanh(tf.matmul(features_mean, w_c) + b_c)

            return c, h

    def _word_embedding(self, inputs, reuse=False):

        with tf.variable_scope('word_embedding', reuse=reuse):

            w = tf.get_variable('w', [self.V, self.M], initializer=self.emb_initializer)

            x = tf.nn.embedding_lookup(w, inputs, name='word_vector')  # (N, T, M) or (N, M)

            return x

    def _project_features(self, features):

        with tf.variable_scope('project_features'):

            w = tf.get_variable('w', [self.D, self.D], initializer=self.weight_initializer)

            features_flat = tf.reshape(features, [-1, self.D])

            features_proj = tf.matmul(features_flat, w)

            features_proj = tf.reshape(features_proj, [-1, self.L, self.D])

            return features_proj

    def _attention_layer(self, features, features_proj, h, reuse=False):

        with tf.variable_scope('attention_layer', reuse=reuse):

            w = tf.get_variable('w', [self.H, self.D], initializer=self.weight_initializer)

            b = tf.get_variable('b', [self.D], initializer=self.const_initializer)

            w_att = tf.get_variable('w_att', [self.D, 1], initializer=self.weight_initializer)

            h_att = tf.nn.relu(features_proj + tf.expand_dims(tf.matmul(h, w), 1) + b)    # (N, L, D)

            out_att = tf.reshape(tf.matmul(tf.reshape(h_att, [-1, self.D]), w_att), [-1, self.L])   # (N, L)

            alpha = tf.nn.softmax(out_att)

            context = tf.reduce_sum(features * tf.expand_dims(alpha, 2), 1, name='context')   #(N, D)

            return context, alpha

    def _selector(self, context, h, reuse=False):

        with tf.variable_scope('selector', reuse=reuse):

            w = tf.get_variable('w', [self.H, 1], initializer=self.weight_initializer)

            b = tf.get_variable('b', [1], initializer=self.const_initializer)

            beta = tf.nn.sigmoid(tf.matmul(h, w) + b, 'beta')    # (N, 1)

            context = tf.multiply(beta, context, name='selected_context')

            return context, beta

    def _decode_lstm(self, x, h, context, dropout=False, reuse=False):

        with tf.variable_scope('logits', reuse=reuse):

            w_h = tf.get_variable('w_h', [self.H, self.M], initializer=self.weight_initializer)

            b_h = tf.get_variable('b_h', [self.M], initializer=self.const_initializer)

            w_out = tf.get_variable('w_out', [self.M, self.V], initializer=self.weight_initializer)

            b_out = tf.get_variable('b_out', [self.V], initializer=self.const_initializer)

            if dropout:

                h = tf.nn.dropout(h, 0.5)

            h_logits = tf.matmul(h, w_h) + b_h

            if self.ctx2out:

                w_ctx2out = tf.get_variable('w_ctx2out', [self.D, self.M], initializer=self.weight_initializer)

                h_logits += tf.matmul(context, w_ctx2out)

            if self.prev2out:

                h_logits += x

            h_logits = tf.nn.tanh(h_logits)

            if dropout:

                h_logits = tf.nn.dropout(h_logits, 0.5)

            out_logits = tf.matmul(h_logits, w_out) + b_out

            return out_logits

    def _batch_norm(self, x, mode='train', name=None):

        return tf.contrib.layers.batch_norm(inputs=x,

                                            decay=0.95,

                                            center=True,

                                            scale=True,

                                            is_training=(mode=='train'),

                                            updates_collections=None,

                                            scope=(name+'batch_norm'))

    def build_model(self):

        features = self.features

        captions = self.captions

        batch_size = tf.shape(features)[0]

        captions_in = captions[:, :self.T]

        captions_out = captions[:, 1:]

        mask = tf.to_float(tf.not_equal(captions_out, self._null))

        # batch normalize feature vectors

        features = self._batch_norm(features, mode='train', name='conv_features')

        c, h = self._get_initial_lstm(features=features)

        x = self._word_embedding(inputs=captions_in)

        features_proj = self._project_features(features=features)

        loss = 0.0

        alpha_list = []

        lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=self.H)

        for t in range(self.T):

            context, alpha = self._attention_layer(features, features_proj, h, reuse=(t!=0))

            alpha_list.append(alpha)

            if self.selector:

                context, beta = self._selector(context, h, reuse=(t!=0))

            with tf.variable_scope('lstm', reuse=(t!=0)):

                _, (c, h) = lstm_cell(inputs=tf.concat( [x[:,t,:], context],1), state=[c, h])

            logits = self._decode_lstm(x[:,t,:], h, context, dropout=self.dropout, reuse=(t!=0))

            loss += tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=captions_out[:, t],logits=logits)*mask[:, t] )

        if self.alpha_c > 0:

            alphas = tf.transpose(tf.stack(alpha_list), (1, 0, 2))     # (N, T, L)

            alphas_all = tf.reduce_sum(alphas, 1)      # (N, L)

            alpha_reg = self.alpha_c * tf.reduce_sum((16./196 - alphas_all) ** 2)

            loss += alpha_reg

        return loss / tf.to_float(batch_size)

    def build_sampler(self, max_len=20):

        features = self.features

        # batch normalize feature vectors

        features = self._batch_norm(features, mode='test', name='conv_features')

        c, h = self._get_initial_lstm(features=features)

        features_proj = self._project_features(features=features)

        sampled_word_list = []

        alpha_list = []

        beta_list = []

        lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=self.H)

        for t in range(max_len):

            if t == 0:

                x = self._word_embedding(inputs=tf.fill([tf.shape(features)[0]], self._start))

            else:

                x = self._word_embedding(inputs=sampled_word, reuse=True)

            context, alpha = self._attention_layer(features, features_proj, h, reuse=(t!=0))

            alpha_list.append(alpha)

            if self.selector:

                context, beta = self._selector(context, h, reuse=(t!=0))

                beta_list.append(beta)

            with tf.variable_scope('lstm', reuse=(t!=0)):

                _, (c, h) = lstm_cell(inputs=tf.concat( [x, context],1), state=[c, h])

            logits = self._decode_lstm(x, h, context, reuse=(t!=0))

            sampled_word = tf.argmax(logits, 1)

            sampled_word_list.append(sampled_word)

        alphas = tf.transpose(tf.stack(alpha_list), (1, 0, 2))     # (N, T, L)

        betas = tf.transpose(tf.squeeze(beta_list), (1, 0))    # (N, T)

        sampled_captions = tf.transpose(tf.stack(sampled_word_list), (1, 0))     # (N, max_len)

        return alphas, betas, sampled_captions

 No newline at end of file

import collections

import tensorflow as tf

from lstm.Readers import helper

import pickle

"""To run this code, you'll need to first download and extract the text dataset

    from here: http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz. Change the

# parser.add_argument('--data_path', type=str, default=data_path, help='The full path of the training data')

# args = parser.parse_args()

"""

with open('interface_train_30pics_per_category.picle', 'rb') as handle:

    data = picle.load(handle)

"""

class basic_masa_reader:

    def read_input(self, filename: str):

        with tf.gfile.GFile(helper.inputPath + filename, "r") as f:

#!/usr/bin/env python3

from lstm.configuration import current as config

import collections

import tensorflow as tf

from lstm.Readers import helper

import pickle

"""To run this code, you'll need to first download and extract the text dataset

    from here: http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz. Change the

    data_path variable below to your local exraction path"""

# data_path = "C:\\Users\Andy\Documents\simple-examples\data"

# parser = argparse.ArgumentParser()

# parser.add_argument('run_opt', type=int, default=1, help='An integer: 1 to train, 2 to test')

# parser.add_argument('--data_path', type=str, default=data_path, help='The full path of the training data')

# args = parser.parse_args()

"""

with open('interface_train_30pics_per_category.picle', 'rb') as handle:

    data = picle.load(handle)

"""

class basic_masa_reader:

    def read_input(self, filename: str):

        with tf.gfile.GFile(helper.inputPath + filename, "r") as f:

            if(config.reader.verbose is True):

                print("")

            return f.read().replace("\n", "<eos>").split()

    def create_vocabulary(self, filename: str):

        data = self.read_input(filename + config.reader.input_data_type)

        counter = collections.Counter(data)

        count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))

        words, _ = list(zip(*count_pairs))

        word_to_id = dict(zip(words, range(len(words))))

        return word_to_id

    def file_to_word_ids(self, filename: str, word_to_id):

        data = self.read_input(filename + config.reader.input_data_type)

        return [word_to_id[word] for word in data if word in word_to_id]

    def load_data(self):

        # build the complete vocabulary, then convert text data to list of integers

        word_to_id = self.create_vocabulary(config.path.input_train)

        train_data = self.file_to_word_ids(config.path.input_train, word_to_id)

        valid_data = self.file_to_word_ids(config.path.input_validate, word_to_id)

        test_data = self.file_to_word_ids(config.path.input_test, word_to_id)

        config.trainer.vocab_size = len(word_to_id)

        reversed_dictionary = dict(zip(word_to_id.values(), word_to_id.keys()))

        if(config.reader.verbose):

            print("\n###### READER ######")

            print("Reader: <train_data>\n", train_data[:20])

            print("Reader: <word_to_id\n", {k: word_to_id[k] for k in list(word_to_id)[:20]})

            print("Reader: <vocab_size>\n", config.trainer.vocab_size)

            print("Reader: <reversed_dictionary>\n", {k: reversed_dictionary[k] for k in list(reversed_dictionary)[:20]})

        return train_data, valid_data, test_data, config.trainer.vocab_size, reversed_dictionary

    def batch_producer(self, raw_data, batch_size, num_steps):

        raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)

        data_len = tf.size(raw_data)

        batch_len = data_len // batch_size

        data = tf.reshape(raw_data[0: batch_size * batch_len],

                        [batch_size, batch_len])

        epoch_size = (batch_len - 1) // num_steps

        i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()

        x = data[:, i * num_steps:(i + 1) * num_steps]

        x.set_shape([batch_size, num_steps])

        y = data[:, i * num_steps + 1: (i + 1) * num_steps + 1]

        y.set_shape([batch_size, num_steps])

        return x, y