lstm.py

import tensorflow as tf
from data_reader import load_data
import numpy as np
from uncompress import *
import os



def one_hot(batch_size,Y):

    B = np.zeros((batch_size,2))

    B[np.arange(batch_size),Y] = 1

    return B


if __name__=='__main__':

    
    # print one_hot(3,np.array((1,0,1)))
    # exit(0)
    # Training Parameters
    learning_rate = 0.001
    num_epoch = 4
    batch_size = 2
    display_step = 1
    input_size = 50
    num_classes = 2 
    lstm_size = 256

    X = tf.placeholder(tf.float32, [None, input_size, 86796])
    Y = tf.placeholder(tf.float32, [None, num_classes])
    # logits = conv_net(X)

    # cell = tf.contrib.rnn.BasicLSTMCell(lstm_size)
    # # print cell.output_shape
    # stacked_lstm = tf.contrib.rnn.MultiRNNCell([cell]*2)

    # print tf.shape(stacked_lstm)
    # exit(0)

    # create 2 LSTMCells
    rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [256, 256]]

    # create a RNN cell composed sequentially of a number of RNNCells
    multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)

    # 'outputs' is a tensor of shape [batch_size, max_time, 256]
    # 'state' is a N-tuple where N is the number of LSTMCells containing a
    # tf.contrib.rnn.LSTMStateTuple for each cell
    output, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell,
                                    inputs=X,
                                    dtype=tf.float32)
  
    
    # data = tf.placeholder(tf.float32, [None, input_size, 86796])
    # output, state = tf.nn.dynamic_rnn(stacked_lstm, X, dtype=tf.float32)
    
    # Select last output.
    output = tf.transpose(output, [1, 0, 2])
    last = tf.gather(output, int(output.get_shape()[0]) - 1)
        
    slim = tf.contrib.slim
    logits = slim.fully_connected(last, 2, scope='fc',activation_fn=None,weights_initializer=tf.truncated_normal_initializer(0.0, 0.01))


    prediction = tf.nn.softmax(logits)
    # # Define loss and optimizer
    loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
         logits=logits, labels=Y))
    tf.summary.scalar('loss',loss_op)
    optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)
    train_op = optimizer.minimize(loss_op)


    # Evaluate model
    correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
    tf.summary.scalar('accuracy',accuracy)

    # Initialize the variables (i.e. assign their default value)
    init = tf.global_variables_initializer()

    data_X,data_Y = load_data()

    indices = np.random.permutation(np.arange(data_X.shape[0]))

    data_X = data_X[indices,:,:]

    data_Y = data_Y[indices]

    

    merged = tf.summary.merge_all()
    saver = tf.train.Saver()

    with tf.Session() as sess:

        train_writer = tf.summary.FileWriter("logs4/",
                                      sess.graph)

        # Run the initializer
        sess.run(init)


        # print 'restoring session'
        # saver.restore(sess, "logs3/epcoh0i0.ckpt")
        # print 'done loading'
        # exit(0)

        i = 0
        print 'started training'
        for epoch in range(num_epoch):
            for step in range(data_X.shape[0]/batch_size):
                batch_x, batch_y = data_X[step*batch_size:(step+1)*batch_size],\
                data_Y[step*batch_size:(step+1)*batch_size]

                

                batch_x = uncompress(batch_x,86796)

                # print batch_y
                batch_y = one_hot(batch_size,batch_y)

                batch_y = np.repeat(batch_y,50,axis=0)

                # print batch_y

                assert(batch_x.shape[0]==batch_y.shape[0])

                # print batch_x.shape
                # print batch_y.shape

                # exit(0)

                # Run optimization op (backprop)
                _,summary = sess.run([train_op,merged], feed_dict={X: batch_x[:,:,:,0], Y: batch_y})
                train_writer.add_summary(summary, i)
                if step % display_step == 0:
                    # Calculate batch loss and accuracy
                    loss, acc,summary = sess.run([loss_op, accuracy,merged], feed_dict={X: batch_x[:,:,:,0],
                                                                        Y: batch_y})
                    print("Epoch : " + str(epoch) + " Step " + str(step) + ", Minibatch Loss= " + \
                        "{:.4f}".format(loss) + ", Training Accuracy= " + \
                        "{:.3f}".format(acc))
                    
                    # train_writer.add_summary(summary, step)
                
                if i%20 == 0:

                    print 'saving checkpoint'
                    save_path = saver.save(sess, os.path.join('logs4','epcoh'+str(epoch)+\
                    'i'+str(i)+'.ckpt'))
                    print("Model saved in path: %s" % save_path)
                
                i+=1

        # print("Optimization Finished!")