added functionality to create folders when they do not excist

parent_folder_path = os.path.dirname(__file__) + "/.."

'''

Check if needed Folders excist if they do not exist create them

'''

def checkFolders():

    if not os.path.exists(parent_folder_path + config.reader.path_source_folder):

        print("[" + str(datetime.now().time()) + "]" + "Reader:: Folder created!\n" + parent_folder_path + config.reader.path_source_folder)

        os.makedirs(parent_folder_path + config.reader.path_source_folder)

    if not os.path.exists(parent_folder_path + config.validator.path_source_folder):

        print("[" + str(datetime.now().time()) + "]" + "Reader:: Folder created!\n" + parent_folder_path + config.validator.path_source_folder)

        os.makedirs(parent_folder_path + config.validator.path_source_folder)

'''

Basic reader to read text files (*.txt)

Edit reader_config in configuration.py

can also be used reformat to our needs

'''

def create_vocabulary(filename):

    checkFolders()

    print("[" + str(datetime.now().time()) + "]" + "Reader:: Creating vocabulary...")

    data = read_input(filename + config.reader.input_data_type,)

    if(config.reader.verbose):

    return word_to_id

'''

Saves created vocabulary into file system

'''

def save_vocabulary(data, filename):

    print("[" + str(datetime.now().time()) + "]" + "Reader:: Saving vocabulary...")

    with open(parent_folder_path + config.validator.path_source_folder + filename + '_vocab' + config.reader.export_data_type, 'w') as outfile:

    if(config.reader.verbose):

        print("Reader:: Saved vocabulary at:", filename)

    print("[" + str(datetime.now().time()) + "]" + "Reader:: ...Finished saving vocabulary!")

"""

def _build_vocab(filename):

  data = read_train_input

  counter = collections.Counter(data)

  count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))

  words, _ = list(zip(*count_pairs))

  word_to_id = dict(zip(words, range(len(words))))

  return word_to_id

"""

"""

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

import collections

import os

import sys

import tensorflow as tf

Py3 = sys.version_info[0] == 3

def _file_to_word_ids(filename, word_to_id):

  data = _read_words(filename)

  return [word_to_id[word] for word in data if word in word_to_id]

def ptb_raw_data(data_path=None):

  train_path = os.path.join(data_path, "ptb.train.txt")

  valid_path = os.path.join(data_path, "ptb.valid.txt")

  test_path = os.path.join(data_path, "ptb.test.txt")

  word_to_id = _build_vocab(train_path)

  train_data = _file_to_word_ids(train_path, word_to_id)

  valid_data = _file_to_word_ids(valid_path, word_to_id)

  test_data = _file_to_word_ids(test_path, word_to_id)

  vocabulary = len(word_to_id)

  return train_data, valid_data, test_data, vocabulary

def ptb_producer(raw_data, batch_size, num_steps, name=None):

  with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):

    raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)

    data_len = tf.size(raw_data)

    batch_len = data_len // batch_size

    data = tf.reshape(raw_data[0 : batch_size * batch_len],

                      [batch_size, batch_len])

    epoch_size = (batch_len - 1) // num_steps

    assertion = tf.assert_positive(

        epoch_size,

        message="epoch_size == 0, decrease batch_size or num_steps")

    with tf.control_dependencies([assertion]):

      epoch_size = tf.identity(epoch_size, name="epoch_size")

    i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()

    x = tf.strided_slice(data, [0, i * num_steps],

                         [batch_size, (i + 1) * num_steps])

    x.set_shape([batch_size, num_steps])

    y = tf.strided_slice(data, [0, i * num_steps + 1],

                         [batch_size, (i + 1) * num_steps + 1])

    y.set_shape([batch_size, num_steps])

    return x, y

"""