train loop added

    input_embedding_size = 20  # length of the charater

    encoder_hidden_units = 20

    decoder_hidden_units = encoder_hidden_units * 2

    max_batches = 300

    batches_in_epoch = 100

        cell_bw=encoder_cell,

        inputs=encoder_inputs_embedded,

        sequence_length=encoder_inputs_length,

        dtype=tf.float64, time_major=True)

        dtype=tf.float32, time_major=True)

    )

    #bidirectional step1

    encoder_max_time, config.trainer.batch_size = tf.unstack(tf.shape(encoder_inputs))

    decoder_lengths = encoder_inputs_length + 3

    #output projects

    #weights and biases

    #SOFT ATTENTION

    W = tf.Variable(tf.random_uniform([config.trainer.decoder_hidden_units], vocab_size), -1, 1), dtype=tf.float32)

    b = tf.Variable(tf.zeroes([config.trainer.vocab_size]), dtype=tf.float32)

    #create padded inputs for the decoder from the word embeddings

    #we are telling the program to test a condition, and trigger an error if the condition is false

    assert config.trainer.end_of_sentence == 1 and config.trainer.padding == 0

    eos_time_slice = tf.ones([batch_size], dtype = tf.int32, name = 'EOS')

    pad_time_slice = tf.ones([batch_size], dtype = tf.int32, name = 'PAD')

    #retrieves rows of the params tensor. The behaviour is similar to using indexing with arrays in numpy

    eos_step_embedded = tf.nn.embedding_lookup(embeddings, eos_time_slice)

    pad_step_embedded = tf.nn.embedding_lookup(embeddings, pad_time_slice)

    #using the functions for the attention decoder

    decoder_outputs_ta, decoder_final_state, _ = tf.nn.raw_rnn(decoder_cell, loop_fn)

    decoder_outputs = decoder_outputs_ta.stack()

    #unpacks the given dimension of a rank-R tensr into a R-1 tensor

    #reduces dimension count

    decoder_max_steps , decoder_batch_size, decoder_dim = tf.unstack(tf.shape(decoder_outputs))

    #flattened output vector

    decoder_outputs_flat = tf.reshape(decoder_outputs, (-1, decoder_dim))

    #pass flattened tensor through decoder

    decoder_logits_flat = tf.add(tf.matmul(decoder_outputs_flat, W), b)

    #prediction values

    #TODO this line of code was not visible correctly in the tutorial

    decoder_logits = tf.reshape(decoder_logits_flat, (decoder_max_steps, decoder_batch_size, config.trainer.vocab_size))

    #final prediction

    decoder_prediction = tf.argmax(decoder_logits, 2)

    # optimizer

    #cross entropy loss

    #one hot encode the target values so we dont rank just differentiate

    stepwise_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(

        labels=tf.one_hot)decoder_targets, depth=config.trainer.vocab_size, dtype=tf.float32),

        logits=decoder_logits

    )

    #loss function

    loss = tf.reduce_mean(stepwise_cross_entropy)

    #train it

    train_op = tf.train.AdamOptimizer().minimize(loss)

    sess.run(tf.global_variables_initializer())

    #training the real deal executes here:

    try:

        for batch in range(config.trainer.max_batches):

            fd = # TODO get a sequence to learn from

            _, l = sess.run([train_op], loss), fd)

            loss_track.append(l)

            if(batch == 0 or batch % config.trainer.batches_in_epoch == 0)

                print('batch {}' .format(batch))

                print('  minibatch loss: {}' .format(sess.run(loss, fd)))

                predict_ = sess.run(decoder_prediction, fd)

                for i, (inp, pred) in enumerate(zip(fd[encoder_inputs].T, predict_.T)):

                    print('    sample {}' .format(i + 1))

                    print('    input     > {}' .format(inp))

                    print('    predicted > {}' .format(pred))

                    if(i >= 2):

                        break

                print()

    except KeyboardInterrupt:

        print('\n.\n.\n.\n...training interupted')

#manually spcifying loop function over time -  to get initial cell state and input to RNN

#normally we would just use dynamic_rnn, but lets get detailed here with raw_rnn

#we define and return these values, no operations occure here

def loop_fn_initial():

    initial_elements_finished = ( 0 >= decoder_lengths)

    #end of sentence

    initial_input = eos_step_embedded

    #last time steps cell state

    initial_cell_state = encoder_final_state

    #none

    initial_cell_output = None

    #none

    initial_loop_state = None

    return( initial_elements_finished,

        initial_input,

        initial_cell_state,

        initial_cell_output,

        initial_loop_state

    )

#attention mechanism --choose which previously generated token to pass as input in the next time step

def loop_fn_transition(time, previous_output, previous_state, previous_loop_state):

    def get_next_input():

        output_logits = tf.add(tf.matmul(previous:output, W), b)

        # this next line is attention

        prediction = tf.argmax(output:logits, axis=1)

        next_inpuut = tf.nn.embedding_lookup(embeddings, prediction)

        return next_inpuut

    elements_finished = (time >= decoder_lengths)

    finished = tf.reduce_all(elements_finished)

    input = tf.cond(finished, lambda: pad_step_embedded, get_next_input)

    state = previous_state

    output = previous_output

    loop_state = None

    return (elements_finished,

        input,

        state,

        output,

        loop_state

    )

def loop_fn(time, previous_output, previous_state, Previous_loop_state):

    if previous_state is None:

        assert previous_output is None and previous_state is None

        return loop_fn_initial()

    else:

        return loop_fn_transition(time, previous_output, previous_state, previous_loop_state)