reworked to class structure and changed lstm algorithm, learning is possible

# reader init

from .basic_text_reader import basic_text_reader

from .basic_text_reader import get_words_as_int

from .basic_text_reader import get_integers_as_words

from .basic_masa_reader import basic_masa_reader

# from .basic_text_reader import basic_text_reader

# from .basic_text_reader import get_words_as_int

# from .basic_text_reader import get_integers_as_words

# from .basic_masa_reader import basic_masa_reader

from .helper import get_input_data

from .helper import get_vocabulary

from .helper import get_sequences

from .helper import check_folders

from .basic_masa_reader import basic_masa_reader

__all__ = [

    'basic_text_reader',

    'basic_masa_reader',

    'get_input_data',

    'get_vocabulary'

    'get_vocabulary',

    'get_sequences',

    'check_folders',

    'basic_masa_reader'

]

#!/usr/bin/env python3

from lstm.configuration import current as config

import collections

import tensorflow as tf

from lstm.Readers import helper

from .. import configuration as config

"""To run this code, you'll need to first download and extract the text dataset

    from here: http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz. Change the

    data_path variable below to your local exraction path"""

# def (config, fileTrain, fileVal, fileTest)

#   formatiern auf usnere domensionen

#   save in config festgelegtem path von e.g. /Data

# data_path = "C:\\Users\Andy\Documents\simple-examples\data"

# parser = argparse.ArgumentParser()

# parser.add_argument('run_opt', type=int, default=1, help='An integer: 1 to train, 2 to test')

# parser.add_argument('--data_path', type=str, default=data_path, help='The full path of the training data')

# args = parser.parse_args()

def basic_masa_reader(file_train, file_val, file_test):

    if(config.current.reader.read_raw_files):

        return True

    return False

class basic_masa_reader:

    def read_input(self, filename: str):

        with tf.gfile.GFile(helper.inputPath + filename, "r") as f:

            return f.read().replace("\n", "<eos>").split()

    def create_vocabulary(self, filename: str):

        data = self.read_input(filename + config.reader.input_data_type)

        counter = collections.Counter(data)

        count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))

        words, _ = list(zip(*count_pairs))

        word_to_id = dict(zip(words, range(len(words))))

        return word_to_id

    def file_to_word_ids(self, filename: str, word_to_id):

        data = self.read_input(filename + config.reader.input_data_type)

        return [word_to_id[word] for word in data if word in word_to_id]

    def load_data(self):

        # build the complete vocabulary, then convert text data to list of integers

        word_to_id = self.create_vocabulary(config.path.input_train)

        train_data = self.file_to_word_ids(config.path.input_train, word_to_id)

        valid_data = self.file_to_word_ids(config.path.input_validate, word_to_id)

        test_data = self.file_to_word_ids(config.path.input_test, word_to_id)

        config.trainer.vocab_size = len(word_to_id)

        reversed_dictionary = dict(zip(word_to_id.values(), word_to_id.keys()))

        if(config.reader.verbose):

            print("\n###### READER ######")

            print("Reader: <train_data>\n", train_data[:20])

            print("Reader: <word_to_id\n", {k: word_to_id[k] for k in list(word_to_id)[:20]})

            print("Reader: <vocab_size>\n", config.trainer.vocab_size)

            print("Reader: <reversed_dictionary>\n", {k: reversed_dictionary[k] for k in list(reversed_dictionary)[:20]})

        return train_data, valid_data, test_data, config.trainer.vocab_size, reversed_dictionary

    def batch_producer(self, raw_data, batch_size, num_steps):

        raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)

        data_len = tf.size(raw_data)

        batch_len = data_len // batch_size

        data = tf.reshape(raw_data[0: batch_size * batch_len],

                        [batch_size, batch_len])

        epoch_size = (batch_len - 1) // num_steps

        i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()

        x = data[:, i * num_steps:(i + 1) * num_steps]

        x.set_shape([batch_size, num_steps])

        y = data[:, i * num_steps + 1: (i + 1) * num_steps + 1]

        y.set_shape([batch_size, num_steps])

        return x, y