others/lab-12-3-many-to-many-keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-3 many to many\n",
    "### simple pos-tagger training \n",
    "* many to many\n",
    "* variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# word의 dictionary\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# pos (part of speech)의 dictionary\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = np.array(list(map(lambda sentence : len(sentence), sentences)))\n",
    "\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence,padding='post')\n",
    "y_data = np.expand_dims(y_data, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<RepeatDataset shapes: ((?, 10), (?, 10, 1), (?, 10)), types: (tf.int32, tf.int32, tf.float32)>\n"
     ]
    }
   ],
   "source": [
    "# input\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data,y_data,x_data_mask))\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2).repeat()\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                                 trainable=False, input_length=max_sequence,\n",
    "                                 embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
    "model.add(layers.TimeDistributed(keras.layers.Dense(units=num_classes, activation='softmax')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10, 10)            260       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             88        \n",
      "=================================================================\n",
      "Total params: 573\n",
      "Trainable params: 348\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a training operation\n",
    "lr = .1\n",
    "opt = tf.train.AdamOptimizer(learning_rate = lr)\n",
    "model.compile(optimizer=opt,\n",
    "              loss=keras.losses.sparse_categorical_crossentropy,\n",
    "              sample_weight_mode=\"temporal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "2/2 [==============================] - 0s 118ms/step - loss: 1.8037\n",
      "Epoch 2/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 1.0356\n",
      "Epoch 3/30\n",
      "2/2 [==============================] - 0s 7ms/step - loss: 0.5860\n",
      "Epoch 4/30\n",
      "2/2 [==============================] - 0s 6ms/step - loss: 0.3096\n",
      "Epoch 5/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 0.1684\n",
      "Epoch 6/30\n",
      "2/2 [==============================] - 0s 6ms/step - loss: 0.0922\n",
      "Epoch 7/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 0.0480\n",
      "Epoch 8/30\n",
      "2/2 [==============================] - 0s 6ms/step - loss: 0.0275\n",
      "Epoch 9/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 0.0150\n",
      "Epoch 10/30\n",
      "2/2 [==============================] - 0s 7ms/step - loss: 0.0089\n",
      "Epoch 11/30\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.0060\n",
      "Epoch 12/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 0.0044\n",
      "Epoch 13/30\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.0032\n",
      "Epoch 14/30\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.0025\n",
      "Epoch 15/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 0.0020\n",
      "Epoch 16/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 0.0016\n",
      "Epoch 17/30\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.0013\n",
      "Epoch 18/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 0.0011\n",
      "Epoch 19/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 0.0010\n",
      "Epoch 20/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 8.9527e-04\n",
      "Epoch 21/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 8.1647e-04\n",
      "Epoch 22/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 7.5537e-04\n",
      "Epoch 23/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 7.0588e-04\n",
      "Epoch 24/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 6.6371e-04\n",
      "Epoch 25/30\n",
      "2/2 [==============================] - 0s 5ms/step - loss: 6.2682e-04\n",
      "Epoch 26/30\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 5.9462e-04\n",
      "Epoch 27/30\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 5.6702e-04\n",
      "Epoch 28/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 5.4372e-04\n",
      "Epoch 29/30\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 5.2418e-04\n",
      "Epoch 30/30\n",
      "2/2 [==============================] - 0s 4ms/step - loss: 5.0758e-04\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f2dbc5d9fd0>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(tr_dataset, epochs=30, steps_per_epoch=4//2, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6. 7. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [4. 7. 2. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [4. 7. 3. 4. 5. 1. 4. 0. 0. 0.]\n",
      " [4. 7. 2. 1. 7. 0. 0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1)\n",
    "print(yhat * x_data_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6, 7, 1, 0, 0, 0, 0, 0, 0, 0],\n",
       "       [4, 7, 2, 1, 0, 0, 0, 0, 0, 0],\n",
       "       [4, 7, 3, 4, 5, 1, 4, 0, 0, 0],\n",
       "       [4, 7, 2, 1, 7, 0, 0, 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.squeeze(y_data, axis=-1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}