working on pythong structure

#reader fiiles listed

#relinking

 No newline at end of file

# reader init

from .basic_masa_reader import basic_masa_reader

from .basic_text_reader import basic_text_reader

 No newline at end of file

#!/usr/bin/env python3

import numpy as np

from .. import configuration as config

#def (config, fileTrain, fileVal, fileTest)

    #formatiern auf usnere domensionen

    #save in config festgelegtem path von e.g. /Data

def basic_masa_reader(file_train, file_val, file_test):

    return None

def read_train_input():

    with tf.gfile.GFile(

        config.current.path_iput_data_folder +

        config.current.path_input_data_train, "r") as f:

            return f.read().split()

def read_validate_input():

    with tf.gfile.GFile( config.path_iput_data_folder + config.path_iput_data_validate, "r") as f:

    if Py3:

      return f.read().replace("\n", "<eos>").split()

    else:

      return f.read().decode("utf-8").replace("\n", "<eos>").split()

def read_test_input():

    with tf.gfile.GFile( config.path_iput_data_folder + config.path_iput_data_test, "r") as f:

    if Py3:

      return f.read().replace("\n", "<eos>").split()

    else:

      return f.read().decode("utf-8").replace("\n", "<eos>").split()

"""Utilities for parsing PTB text files."""

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

import collections

import os

import sys

import tensorflow as tf

Py3 = sys.version_info[0] == 3

def _read_words(filename):

  with tf.gfile.GFile(filename, "r") as f:

    if Py3:

      return f.read().replace("\n", "<eos>").split()

    else:

      return f.read().decode("utf-8").replace("\n", "<eos>").split()

def _build_vocab(filename):

  data = _read_words(filename)

  counter = collections.Counter(data)

  count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))

  words, _ = list(zip(*count_pairs))

  word_to_id = dict(zip(words, range(len(words))))

  return word_to_id

def _file_to_word_ids(filename, word_to_id):

  data = _read_words(filename)

  return [word_to_id[word] for word in data if word in word_to_id]

def ptb_raw_data(data_path=None):

  """Load PTB raw data from data directory "data_path".

  Reads PTB text files, converts strings to integer ids,

  and performs mini-batching of the inputs.

  The PTB dataset comes from Tomas Mikolov's webpage:

  http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz

  Args:

    data_path: string path to the directory where simple-examples.tgz has

      been extracted.

  Returns:

    tuple (train_data, valid_data, test_data, vocabulary)

    where each of the data objects can be passed to PTBIterator.

  """

  train_path = os.path.join(data_path, "ptb.train.txt")

  valid_path = os.path.join(data_path, "ptb.valid.txt")

  test_path = os.path.join(data_path, "ptb.test.txt")

  word_to_id = _build_vocab(train_path)

  train_data = _file_to_word_ids(train_path, word_to_id)

  valid_data = _file_to_word_ids(valid_path, word_to_id)

  test_data = _file_to_word_ids(test_path, word_to_id)

  vocabulary = len(word_to_id)

  return train_data, valid_data, test_data, vocabulary

def ptb_producer(raw_data, batch_size, num_steps, name=None):

  """Iterate on the raw PTB data.

  This chunks up raw_data into batches of examples and returns Tensors that

  are drawn from these batches.

  Args:

    raw_data: one of the raw data outputs from ptb_raw_data.

    batch_size: int, the batch size.

    num_steps: int, the number of unrolls.

    name: the name of this operation (optional).

  Returns:

    A pair of Tensors, each shaped [batch_size, num_steps]. The second element

    of the tuple is the same data time-shifted to the right by one.

  Raises:

    tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.

  """

  with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):

    raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)

    data_len = tf.size(raw_data)

    batch_len = data_len // batch_size

    data = tf.reshape(raw_data[0 : batch_size * batch_len],

                      [batch_size, batch_len])

    epoch_size = (batch_len - 1) // num_steps

    assertion = tf.assert_positive(

        epoch_size,

        message="epoch_size == 0, decrease batch_size or num_steps")

    with tf.control_dependencies([assertion]):

      epoch_size = tf.identity(epoch_size, name="epoch_size")

    i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()

    x = tf.strided_slice(data, [0, i * num_steps],

                         [batch_size, (i + 1) * num_steps])

    x.set_shape([batch_size, num_steps])

    y = tf.strided_slice(data, [0, i * num_steps + 1],

                         [batch_size, (i + 1) * num_steps + 1])

    y.set_shape([batch_size, num_steps])

    return x, y

 No newline at end of file

#!/usr/bin/env python3

from .. import configuration as config

#def (config, fileTrain, fileVal, fileTest)

    #formatiern auf usnere domensionen

    #save in config festgelegtem path von e.g. /Data

def basic_text_reader():

    read_train_input()

    read_validate_input

    read_test_input

    return None

def read_train_input():

    with tf.gfile.GFile(config.current.path_iput_data_folder + config.current.path_input_data_train, "r") as f:

        return f.read().split()

def read_validate_input():

    with tf.gfile.GFile(config.current.path_iput_data_folder + config.current.path_input_data_validate, "r") as f:

        return f.read().split()

def read_test_input():

    with tf.gfile.GFile(config.current.path_iput_data_folder + config.current.path_input_data_test, "r") as f:

        return f.read().split()

def _build_vocab(filename):

  data = _read_words(filename)

  counter = collections.Counter(data)

  count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))

  words, _ = list(zip(*count_pairs))

  word_to_id = dict(zip(words, range(len(words))))

  return word_to_id

"""Utilities for parsing PTB text files."""

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

import collections

import os

import sys

import tensorflow as tf

Py3 = sys.version_info[0] == 3

def _file_to_word_ids(filename, word_to_id):

  data = _read_words(filename)

  return [word_to_id[word] for word in data if word in word_to_id]

def ptb_raw_data(data_path=None):

  """Load PTB raw data from data directory "data_path".

  Reads PTB text files, converts strings to integer ids,

  and performs mini-batching of the inputs.

  The PTB dataset comes from Tomas Mikolov's webpage:

  http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz

  Args:

    data_path: string path to the directory where simple-examples.tgz has

      been extracted.

  Returns:

    tuple (train_data, valid_data, test_data, vocabulary)

    where each of the data objects can be passed to PTBIterator.

  """

  train_path = os.path.join(data_path, "ptb.train.txt")

  valid_path = os.path.join(data_path, "ptb.valid.txt")

  test_path = os.path.join(data_path, "ptb.test.txt")

  word_to_id = _build_vocab(train_path)

  train_data = _file_to_word_ids(train_path, word_to_id)

  valid_data = _file_to_word_ids(valid_path, word_to_id)

  test_data = _file_to_word_ids(test_path, word_to_id)

  vocabulary = len(word_to_id)

  return train_data, valid_data, test_data, vocabulary

def ptb_producer(raw_data, batch_size, num_steps, name=None):

  """Iterate on the raw PTB data.

  This chunks up raw_data into batches of examples and returns Tensors that

  are drawn from these batches.

  Args:

    raw_data: one of the raw data outputs from ptb_raw_data.

    batch_size: int, the batch size.

    num_steps: int, the number of unrolls.

    name: the name of this operation (optional).

  Returns:

    A pair of Tensors, each shaped [batch_size, num_steps]. The second element

    of the tuple is the same data time-shifted to the right by one.

  Raises:

    tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.

  """

  with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):

    raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)

    data_len = tf.size(raw_data)

    batch_len = data_len // batch_size

    data = tf.reshape(raw_data[0 : batch_size * batch_len],

                      [batch_size, batch_len])

    epoch_size = (batch_len - 1) // num_steps

    assertion = tf.assert_positive(

        epoch_size,

        message="epoch_size == 0, decrease batch_size or num_steps")

    with tf.control_dependencies([assertion]):

      epoch_size = tf.identity(epoch_size, name="epoch_size")

    i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()

    x = tf.strided_slice(data, [0, i * num_steps],

                         [batch_size, (i + 1) * num_steps])

    x.set_shape([batch_size, num_steps])

    y = tf.strided_slice(data, [0, i * num_steps + 1],

                         [batch_size, (i + 1) * num_steps + 1])

    y.set_shape([batch_size, num_steps])

    return x, y

 No newline at end of file

#validator files listed

#relinking

 No newline at end of file

# validator init

from .basic_mscoco_validator import basic_mscoco_validator

 No newline at end of file

#   lassen wir das testDatenSet durch unser model laufen

#   evaluieren mit unserer validierungsfunktion

#       (validator)

def basic_mscoco_validator(config, test_daten_set):

    return None