Commit 302fbc6c authored by Rudolf Chrispens's avatar Rudolf Chrispens
Browse files

refactoring reader and adding some helpers also starting tensorflow trainer

parent 197f049a

caption-lib/lstm/Readers/__init__.py

+5 −1
Original line number Diff line number Diff line 
# reader init

from .basic_text_reader import basic_text_reader

from .basic_masa_reader import basic_masa_reader

from .helper import get_input_data

from .helper import get_vocabulary



__all__ = [

    'basic_text_reader',

    'basic_masa_reader'

    'basic_masa_reader',

    'get_input_data',

    'get_vocabulary'

]
 
 No newline at end of file

caption-lib/lstm/Readers/basic_text_reader.py

+18 −17
Original line number Diff line number Diff line
@@ -8,18 +8,20 @@ import os 
from datetime import datetime



parent_folder_path = os.path.dirname(__file__) + "/.."

inputPath = parent_folder_path + config.path.input_folder_path

outputPath = parent_folder_path + config.path.read_folder_path



'''

Check if needed Folders excist if they do not exist create them

'''

def checkFolders():

    if not os.path.exists(parent_folder_path + config.reader.path_source_folder):

        print("[" + str(datetime.now().time()) + "]" + "Reader:: Folder created!\n" + parent_folder_path + config.reader.path_source_folder)

        os.makedirs(parent_folder_path + config.reader.path_source_folder)

    if not os.path.exists(inputPath):

        print("[" + str(datetime.now().time()) + "]" + "Reader:: Folder created!\n" + inputPath)

        os.makedirs(inputPath)



    if not os.path.exists(parent_folder_path + config.validator.path_source_folder):

        print("[" + str(datetime.now().time()) + "]" + "Reader:: Folder created!\n" + parent_folder_path + config.validator.path_source_folder)

        os.makedirs(parent_folder_path + config.validator.path_source_folder)

    if not os.path.exists(outputPath):

        print("[" + str(datetime.now().time()) + "]" + "Reader:: Folder created!\n" + outputPath)

        os.makedirs(outputPath)





'''
@@ -28,24 +30,23 @@ Edit reader_config in configuration.py 
'''

def basic_text_reader():

    if(config.reader.read_raw_files):

        read_input(config.reader.input_train)

        read_input(config.reader.input_validate)

        read_input(config.reader.input_test)

        read_input(config.path.input_train)

        read_input(config.path.input_validate)

        read_input(config.path.input_test)



    if(config.reader.create_vocab):

        create_vocabulary(config.reader.input_train)

        create_vocabulary(config.reader.input_validate)

        create_vocabulary(config.reader.input_test)

        create_vocabulary(config.path.input_train)

        create_vocabulary(config.path.input_validate)

        create_vocabulary(config.path.input_test)





'''

Read files

Read files with split()

'''

def read_input(filename):

    with tf.gfile.GFile(parent_folder_path + config.reader.path_source_folder + filename, "r") as f:

    with tf.gfile.GFile(inputPath + filename, "r") as f:

        return f.read().split()





'''

Create vocabulary while reading files

can also be used reformat to our needs
@@ -54,7 +55,7 @@ def create_vocabulary(filename): 
    checkFolders()



    print("[" + str(datetime.now().time()) + "]" + "Reader:: Creating vocabulary...")

    data = read_input(filename + config.reader.input_data_type,)

    data = read_input(filename + config.reader.input_data_type)

    if(config.reader.verbose):

        print("Reader:: Read Sample 1:", data[0])
@@ -78,7 +79,7 @@ Saves created vocabulary into file system 
'''

def save_vocabulary(data, filename):

    print("[" + str(datetime.now().time()) + "]" + "Reader:: Saving vocabulary...")

    with open(parent_folder_path + config.validator.path_source_folder + filename + '_vocab' + config.reader.export_data_type, 'w') as outfile:

    with open(parent_folder_path + config.path.read_folder_path + filename + '_vocab' + config.reader.export_data_type, 'w') as outfile:

        jsonlib.dump(data, outfile)



    if(config.reader.verbose):

caption-lib/lstm/Readers/helper.py

0 → 100644
+18 −0
Original line number Diff line number Diff line 


import os

import tensorflow as tf

import json as jsonlib

from lstm.configuration import current as config





parent_folder_path = os.path.dirname(__file__) + "/.."

inputPath = parent_folder_path + config.path.input_folder_path

outputPath = parent_folder_path + config.path.read_folder_path



def get_input_data(filename):

    with tf.gfile.GFile(inputPath + filename, "r") as f:

        return f.read()



def get_vocabulary(filename):

    with open(outputPath + filename + '_vocab' + config.reader.export_data_type, 'r') as outfile:

        return jsonlib.load(outfile)

caption-lib/lstm/configuration.py

+28 −9
Original line number Diff line number Diff line
@@ -5,23 +5,34 @@ DEFAULT 
do not change the default settings if possible!

use the Current() configuration to change the settings!

'''

class default_reader(object):  # default reader config

    path_source_folder = "/Input_Data"

    input_data_type = ".txt"

class default_paths(object):

    # input

    input_folder_path = "/Input_Data"

    input_train = "/train"

    input_validate = "/validate"

    input_test = "/test"

    # read already and for e.g. rewritten to vocab?

    read_folder_path = "/Read_Data"

    read_train = "/train"

    read_validate = "/validate"

    read_test = "/test"



class default_reader(object):  # default reader config

    reader_on = False

    input_data_type = ".txt"

    export_data_type = ".json"

    verbose = False

    read_raw_files = False  # this will only read files use only for testing

    create_vocab = True





class default_validator(object):

    path_source_folder = "/Read_Data"

    read_train = "/train.json"

    read_validate = "/validate.json"

    read_test = "/test.json"

    validator_on = True

    input_data_type = ".json"

    



class default_trainer(object):

    train_on = True

    



'''

CURRENT SET CONFIGURATION
@@ -29,9 +40,17 @@ use this to reconfigure the default configurations 
NOTE: later on there will be args[] available to configure the current.

NOTE: later on there will be other settings like small/medium/high that can override current!

'''

class current(default_reader, default_validator):  # Custom presets at bottom

class current(default_reader, default_validator, default_trainer, default_paths):  # Custom presets at bottom

    path = default_paths

    

    reader = default_reader

    reader.reader_on = False

    reader.verbose = False

    # e.g. reader.input_train = "/train2"

    

    validator = default_validator

    validator.validator_on = False

    # e.g. validator.read_test = "/test1234"

    

    trainer = default_trainer

    trainer.train_on = True
 
 No newline at end of file

caption-lib/lstm/lstm_main.py

+6 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3



import lstm.Readers as reader

from lstm.configuration import current as config

import lstm.lstm_train as trainer

# import Validators as validator



def main(self, parameter_list):

    if(config.reader.reader_on):

        reader.basic_text_reader()



    if(config.trainer.train_on):

        trainer.train()

    

# class arguments



# main