attention/2.luong.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from utils import *\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ONEHOT = np.zeros((len(trainset.data),len(trainset.target_names)))\n",
    "ONEHOT[np.arange(len(trainset.data)),trainset.target] = 1.0\n",
    "train_X, test_X, train_Y, test_Y, train_onehot, test_onehot = train_test_split(trainset.data, \n",
    "                                                                               trainset.target, \n",
    "                                                                               ONEHOT, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 20332\n",
      "Most common words [('film', 1453), ('movie', 1270), ('one', 727), ('like', 721), ('story', 477), ('much', 386)]\n",
      "Sample data [562, 2617, 3517, 16582, 36, 9639, 217, 152, 19, 4369] ['rock', 'destined', '21st', 'centurys', 'new', 'conan', 'hes', 'going', 'make', 'splash']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(trainset.data).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention:\n",
    "    def __init__(self,hidden_size):\n",
    "        self.hidden_size = hidden_size\n",
    "        self.dense_layer = tf.layers.Dense(hidden_size)\n",
    "        self.v = tf.random_normal([hidden_size],mean=0,stddev=1/np.sqrt(hidden_size))\n",
    "        \n",
    "    def score(self, hidden_tensor, encoder_outputs):\n",
    "        energy = tf.nn.tanh(self.dense_layer(encoder_outputs))\n",
    "        energy = tf.transpose(energy,[0,2,1])\n",
    "        batch_size = tf.shape(encoder_outputs)[0]\n",
    "        v = tf.expand_dims(tf.tile(tf.expand_dims(self.v,0),[batch_size,1]),1)\n",
    "        energy = tf.matmul(v,energy)\n",
    "        return tf.squeeze(energy,1)\n",
    "    \n",
    "    def __call__(self, hidden, encoder_outputs):\n",
    "        seq_len = tf.shape(encoder_outputs)[1]\n",
    "        batch_size = tf.shape(encoder_outputs)[0]\n",
    "        H = tf.tile(tf.expand_dims(hidden, 1),[1,seq_len,1])\n",
    "        attn_energies = self.score(H,encoder_outputs)\n",
    "        return tf.expand_dims(tf.nn.softmax(attn_energies),1)\n",
    "\n",
    "class Luong(tf.contrib.rnn.RNNCell):\n",
    "    def __init__(self, hidden_size, output_size, encoder_outputs):\n",
    "        self.hidden_size = hidden_size\n",
    "        self.batch_size = tf.shape(encoder_outputs)[0]\n",
    "        self.gru = tf.contrib.rnn.GRUCell(hidden_size)\n",
    "        self.attention = Attention(hidden_size)\n",
    "        self.out = tf.layers.Dense(output_size)\n",
    "        self.encoder_outputs = encoder_outputs\n",
    "        self.reset_state()\n",
    "    \n",
    "    @property\n",
    "    def state_size(self):\n",
    "        return self.hidden_size\n",
    " \n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return self.hidden_size\n",
    "    \n",
    "    def reset_state(self):\n",
    "        self.context = tf.zeros(shape=(self.batch_size,self.hidden_size))\n",
    "    \n",
    "    def __call__(self,inputs, state, scope=None):\n",
    "        rnn_input = tf.concat([inputs,self.context],1)\n",
    "        output, hidden = self.gru(rnn_input, state)\n",
    "        attn_weights = self.attention(output, self.encoder_outputs)\n",
    "        self.context = tf.matmul(attn_weights, self.encoder_outputs)[:,0,:]\n",
    "        output = tf.concat([output,self.context],1)\n",
    "        output = tf.nn.softmax(self.out(output))\n",
    "        return output, hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, embedded_size, dict_size, dimension_output, learning_rate):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.float32, [None, dimension_output])\n",
    "        self.encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(self.encoder_embeddings, self.X)\n",
    "        self.luong_cell = Luong(size_layer,size_layer,encoder_embedded)\n",
    "        outputs, last_states = tf.nn.dynamic_rnn(self.luong_cell, encoder_embedded, dtype = tf.float32)\n",
    "        W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.matmul(outputs[:,-1], W) + b\n",
    "        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-7-50dce70c2347>:13: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "size_layer = 128\n",
    "embedded_size = 128\n",
    "dimension_output = len(trainset.target_names)\n",
    "learning_rate = 1e-3\n",
    "maxlen = 50\n",
    "batch_size = 128\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,embedded_size,len(dictionary),dimension_output,learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.540039\n",
      "time taken: 9.742317914962769\n",
      "epoch: 0, training loss: 0.692666, training acc: 0.503551, valid loss: 0.688301, valid acc: 0.540039\n",
      "\n",
      "epoch: 1, pass acc: 0.540039, current acc: 0.660645\n",
      "time taken: 9.577836036682129\n",
      "epoch: 1, training loss: 0.653488, training acc: 0.645952, valid loss: 0.636614, valid acc: 0.660645\n",
      "\n",
      "epoch: 2, pass acc: 0.660645, current acc: 0.691895\n",
      "time taken: 9.636506795883179\n",
      "epoch: 2, training loss: 0.561013, training acc: 0.780777, valid loss: 0.610502, valid acc: 0.691895\n",
      "\n",
      "time taken: 9.737434148788452\n",
      "epoch: 3, training loss: 0.484725, training acc: 0.844579, valid loss: 0.607925, valid acc: 0.687988\n",
      "\n",
      "epoch: 4, pass acc: 0.691895, current acc: 0.698730\n",
      "time taken: 9.770654678344727\n",
      "epoch: 4, training loss: 0.428225, training acc: 0.876539, valid loss: 0.606709, valid acc: 0.698730\n",
      "\n",
      "time taken: 9.933742046356201\n",
      "epoch: 5, training loss: 0.371753, training acc: 0.911695, valid loss: 0.612761, valid acc: 0.698242\n",
      "\n",
      "epoch: 6, pass acc: 0.698730, current acc: 0.701172\n",
      "time taken: 10.066709518432617\n",
      "epoch: 6, training loss: 0.325556, training acc: 0.931226, valid loss: 0.617917, valid acc: 0.701172\n",
      "\n",
      "time taken: 10.072763204574585\n",
      "epoch: 7, training loss: 0.291312, training acc: 0.942590, valid loss: 0.636979, valid acc: 0.699707\n",
      "\n",
      "epoch: 8, pass acc: 0.701172, current acc: 0.701660\n",
      "time taken: 10.197040557861328\n",
      "epoch: 8, training loss: 0.267891, training acc: 0.947325, valid loss: 0.644354, valid acc: 0.701660\n",
      "\n",
      "epoch: 9, pass acc: 0.701660, current acc: 0.705078\n",
      "time taken: 10.20655870437622\n",
      "epoch: 9, training loss: 0.245343, training acc: 0.953480, valid loss: 0.652071, valid acc: 0.705078\n",
      "\n",
      "time taken: 13.8681321144104\n",
      "epoch: 10, training loss: 0.228671, training acc: 0.957150, valid loss: 0.667429, valid acc: 0.705078\n",
      "\n",
      "epoch: 11, pass acc: 0.705078, current acc: 0.708984\n",
      "time taken: 20.409197330474854\n",
      "epoch: 11, training loss: 0.215550, training acc: 0.959635, valid loss: 0.676031, valid acc: 0.708984\n",
      "\n",
      "time taken: 20.692713260650635\n",
      "epoch: 12, training loss: 0.203894, training acc: 0.961293, valid loss: 0.693618, valid acc: 0.705078\n",
      "\n",
      "time taken: 20.51321005821228\n",
      "epoch: 13, training loss: 0.194636, training acc: 0.962831, valid loss: 0.710977, valid acc: 0.704102\n",
      "\n",
      "time taken: 20.729293823242188\n",
      "epoch: 14, training loss: 0.187575, training acc: 0.963305, valid loss: 0.726808, valid acc: 0.702637\n",
      "\n",
      "time taken: 20.588152647018433\n",
      "epoch: 15, training loss: 0.181914, training acc: 0.963187, valid loss: 0.742926, valid acc: 0.699219\n",
      "\n",
      "time taken: 20.96327304840088\n",
      "epoch: 16, training loss: 0.174368, training acc: 0.965199, valid loss: 0.743632, valid acc: 0.708496\n",
      "\n",
      "break epoch:17\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)\n",
    "        model.luong_cell.reset_state()\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_onehot[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)\n",
    "        model.luong_cell.reset_state()\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_onehot[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.71      0.68      0.70      1068\n",
      "   positive       0.69      0.72      0.71      1065\n",
      "\n",
      "avg / total       0.70      0.70      0.70      2133\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.luong_cell.reset_state()\n",
    "logits = sess.run(model.logits, feed_dict={model.X:str_idx(test_X,dictionary,maxlen)})\n",
    "print(metrics.classification_report(test_Y, np.argmax(logits,1), target_names = trainset.target_names))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}