origin_source/lab-12-0-rnn_basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sinseongjin/tf110/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "# http://www.wildml.com/2016/08/rnns-in-tensorflow-a-practical-guide-and-undocumented-features/\n",
    "# http://learningtensorflow.com/index.html\n",
    "# http://suriyadeepan.github.io/2016-12-31-practical-seq2seq/\n",
    "\n",
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.contrib import rnn\n",
    "import pprint\n",
    "\n",
    "# tf.enable_eager_execution() # Use eager instead of Interactive session\n",
    "sess = tf.InteractiveSession()\n",
    "pp = pprint.PrettyPrinter(indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One hot encoding for each char in 'hello'\n",
    "h = [1, 0, 0, 0]\n",
    "e = [0, 1, 0, 0]\n",
    "l = [0, 0, 1, 0]\n",
    "o = [0, 0, 0, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23348727/cc981856-fce7-11e6-83ea-4b187473466b.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'cell output size: 2, cell state size: 2'\n",
      "'x_data: [[[1. 0. 0. 0.]]]'\n",
      "'outputs: [[[-0.14435719  0.03712483]]]'\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('one_cell') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (2)\n",
    "    hidden_size = 2\n",
    "    cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size) #Using RNN\n",
    "#     cell = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size) # Using LSTM\n",
    "#     cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_size) # Using GRU\n",
    "    pp.pprint(\"cell output size: {}, cell state size: {}\".format(cell.output_size, cell.state_size))\n",
    "\n",
    "    x_data = np.array([[h]], dtype=np.float32) # x_data = [[[1,0,0,0]]]\n",
    "    pp.pprint(\"x_data: {}\".format(x_data))\n",
    "    \n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n",
    "    \n",
    "    pp.pprint(\"outputs: {}\".format(outputs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23383634/649efd0a-fd82-11e6-925d-8041242743b0.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'x_data shape: (1, 5, 4)'\n",
      "('x_data: [[[1. 0. 0. 0.]\\n'\n",
      " '  [0. 1. 0. 0.]\\n'\n",
      " '  [0. 0. 1. 0.]\\n'\n",
      " '  [0. 0. 1. 0.]\\n'\n",
      " '  [0. 0. 0. 1.]]]')\n",
      "('outputs: [[[-0.11998465  0.03867375]\\n'\n",
      " '  [ 0.06975887  0.15671481]\\n'\n",
      " '  [ 0.14599895  0.10636127]\\n'\n",
      " '  [ 0.19630006  0.07254043]\\n'\n",
      " '  [ 0.17188339  0.03171819]]]')\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('two_sequances') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (2). sequence: 5\n",
    "    hidden_size = 2\n",
    "    cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size)\n",
    "#     cell = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size) # Using LSTM\n",
    "#     cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_size) # Using GRU\n",
    "    \n",
    "    x_data = np.array([[h, e, l, l, o]], dtype=np.float32)\n",
    "\n",
    "    pp.pprint(\"x_data shape: {}\".format(x_data.shape))\n",
    "    pp.pprint(\"x_data: {}\".format(x_data))\n",
    "    \n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n",
    "    \n",
    "    pp.pprint(\"outputs: {}\".format(outputs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Work in Progress"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image](https://cloud.githubusercontent.com/assets/901975/23383681/9943a9fc-fd82-11e6-8121-bd187994e249.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[1., 0., 0., 0.],\n",
      "        [0., 1., 0., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 0., 1.]],\n",
      "\n",
      "       [[0., 1., 0., 0.],\n",
      "        [0., 0., 0., 1.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.]],\n",
      "\n",
      "       [[0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 1., 0., 0.],\n",
      "        [0., 1., 0., 0.],\n",
      "        [0., 0., 1., 0.]]], dtype=float32)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'sess' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-743eee05c30c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m     outputs, _states = tf.nn.dynamic_rnn(\n\u001b[1;32m     13\u001b[0m         cell, x_data, dtype=tf.float32)\n\u001b[0;32m---> 14\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0mpp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sess' is not defined"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('3_batches') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (2). sequence: 5, batch 3\n",
    "    # 3 batches 'hello', 'eolll', 'lleel'\n",
    "    x_data = np.array([[h, e, l, l, o],\n",
    "                       [e, o, l, l, l],\n",
    "                       [l, l, e, e, l]], dtype=np.float32)\n",
    "    pp.pprint(x_data)\n",
    "    \n",
    "    hidden_size = 2\n",
    "    cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n",
    "#     cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_size) # Using GRU\n",
    "    outputs, _states = tf.nn.dynamic_rnn(\n",
    "        cell, x_data, dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[1., 0., 0., 0.],\n",
      "        [0., 1., 0., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 0., 1.]],\n",
      "\n",
      "       [[0., 1., 0., 0.],\n",
      "        [0., 0., 0., 1.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.]],\n",
      "\n",
      "       [[0., 0., 1., 0.],\n",
      "        [0., 0., 1., 0.],\n",
      "        [0., 1., 0., 0.],\n",
      "        [0., 1., 0., 0.],\n",
      "        [0., 0., 1., 0.]]], dtype=float32)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'sess' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-afd53d56ea3d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m     outputs, _states = tf.nn.dynamic_rnn(\n\u001b[1;32m     13\u001b[0m         cell, x_data, sequence_length=[5,3,4], dtype=tf.float32)\n\u001b[0;32m---> 14\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0mpp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sess' is not defined"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('3_batches_dynamic_length') as scope:\n",
    "    # One cell RNN input_dim (4) -> output_dim (5). sequence: 5, batch 3\n",
    "    # 3 batches 'hello', 'eolll', 'lleel'\n",
    "    x_data = np.array([[h, e, l, l, o],\n",
    "                       [e, o, l, l, l],\n",
    "                       [l, l, e, e, l]], dtype=np.float32)\n",
    "    pp.pprint(x_data)\n",
    "    \n",
    "    hidden_size = 2\n",
    "    cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n",
    "    \n",
    "    outputs, _states = tf.nn.dynamic_rnn(\n",
    "        cell, x_data, sequence_length=[5,3,4], dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[ 1.,  0.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.]],\n",
      "\n",
      "       [[ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  0.,  1.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]],\n",
      "\n",
      "       [[ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  1.,  0.,  0.],\n",
      "        [ 0.,  0.,  1.,  0.]]], dtype=float32)\n",
      "array([[[-0.10699597, -0.06735446],\n",
      "        [-0.01214837, -0.03093243],\n",
      "        [-0.11962769,  0.10190663],\n",
      "        [-0.20870663,  0.20675965],\n",
      "        [-0.0284863 ,  0.1948234 ]],\n",
      "\n",
      "       [[ 0.06502327,  0.01373457],\n",
      "        [ 0.11152857,  0.07884337],\n",
      "        [-0.00791237,  0.20213564],\n",
      "        [-0.14327727,  0.27671731],\n",
      "        [-0.2389248 ,  0.32179514]],\n",
      "\n",
      "       [[-0.11509674,  0.13784541],\n",
      "        [-0.20916083,  0.231289  ],\n",
      "        [-0.12755771,  0.12080902],\n",
      "        [-0.03355049,  0.0995911 ],\n",
      "        [-0.14717762,  0.24482173]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('initial_state') as scope:\n",
    "    batch_size = 3\n",
    "    x_data = np.array([[h, e, l, l, o],\n",
    "                      [e, o, l, l, l],\n",
    "                      [l, l, e, e, l]], dtype=np.float32)\n",
    "    pp.pprint(x_data)\n",
    "    \n",
    "    # One cell RNN input_dim (4) -> output_dim (5). sequence: 5, batch: 3\n",
    "    hidden_size=2\n",
    "    cell = rnn.BasicLSTMCell(num_units=hidden_size, state_is_tuple=True)\n",
    "    initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data,\n",
    "                                         initial_state=initial_state, dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[ 0.,  1.,  2.],\n",
      "        [ 3.,  4.,  5.],\n",
      "        [ 6.,  7.,  8.],\n",
      "        [ 9., 10., 11.],\n",
      "        [12., 13., 14.]],\n",
      "\n",
      "       [[15., 16., 17.],\n",
      "        [18., 19., 20.],\n",
      "        [21., 22., 23.],\n",
      "        [24., 25., 26.],\n",
      "        [27., 28., 29.]],\n",
      "\n",
      "       [[30., 31., 32.],\n",
      "        [33., 34., 35.],\n",
      "        [36., 37., 38.],\n",
      "        [39., 40., 41.],\n",
      "        [42., 43., 44.]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Create input data\n",
    "batch_size=3\n",
    "sequence_length=5\n",
    "input_dim=3\n",
    "\n",
    "x_data = np.arange(45, dtype=np.float32).reshape(batch_size, sequence_length, input_dim)\n",
    "pp.pprint(x_data)  # batch, sequence_length, input_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sess' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-0dbfe08e9fe5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m     outputs, _states = tf.nn.dynamic_rnn(cell, x_data,\n\u001b[1;32m      6\u001b[0m                                          initial_state=initial_state, dtype=tf.float32)\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mpp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sess' is not defined"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('generated_data') as scope:\n",
    "    # One cell RNN input_dim (3) -> output_dim (5). sequence: 5, batch: 3\n",
    "    cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    initial_state = cell.zero_state(batch_size, tf.float32)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data,\n",
    "                                         initial_state=initial_state, dtype=tf.float32)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "InvalidArgumentError",
     "evalue": "Matrix size-incompatible: In[0]: [3,10], In[1]: [8,20] [Op:MatMul] name: MultiRNNCell/rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/MatMul/",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-877a50c16b97>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;31m# rnn in/out\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdynamic_rnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"dynamic rnn: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36mdynamic_rnn\u001b[0;34m(cell, inputs, sequence_length, initial_state, dtype, parallel_iterations, swap_memory, time_major, scope)\u001b[0m\n\u001b[1;32m    629\u001b[0m         \u001b[0mswap_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswap_memory\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    630\u001b[0m         \u001b[0msequence_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msequence_length\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 631\u001b[0;31m         dtype=dtype)\n\u001b[0m\u001b[1;32m    632\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m     \u001b[0;31m# Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36m_dynamic_rnn_loop\u001b[0;34m(cell, inputs, initial_state, parallel_iterations, swap_memory, sequence_length, dtype)\u001b[0m\n\u001b[1;32m    826\u001b[0m       \u001b[0mparallel_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparallel_iterations\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m       \u001b[0mmaximum_iterations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m       swap_memory=swap_memory)\n\u001b[0m\u001b[1;32m    829\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m   \u001b[0;31m# Unpack final output if not using output tuples.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36mwhile_loop\u001b[0;34m(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name, maximum_iterations, return_same_structure)\u001b[0m\n\u001b[1;32m   3206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3207\u001b[0m       \u001b[0;32mwhile\u001b[0m \u001b[0mcond\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mloop_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3208\u001b[0;31m         \u001b[0mloop_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mloop_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3209\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtry_to_pack\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloop_vars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_basetuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3210\u001b[0m           \u001b[0mpacked\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(i, lv)\u001b[0m\n\u001b[1;32m   3199\u001b[0m         cond = lambda i, lv: (  # pylint: disable=g-long-lambda\n\u001b[1;32m   3200\u001b[0m             math_ops.logical_and(i < maximum_iterations, orig_cond(*lv)))\n\u001b[0;32m-> 3201\u001b[0;31m         \u001b[0mbody\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlv\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morig_body\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mlv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3203\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36m_time_step\u001b[0;34m(time, output_ta_t, state)\u001b[0m\n\u001b[1;32m    797\u001b[0m           skip_conditionals=True)\n\u001b[1;32m    798\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 799\u001b[0;31m       \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    800\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    801\u001b[0m     \u001b[0;31m# Pack state if using state tuples\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    783\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    784\u001b[0m     \u001b[0minput_t\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpack_sequence_as\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstructure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflat_sequence\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 785\u001b[0;31m     \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_t\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m    220\u001b[0m         \u001b[0msetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope_attrname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mscope\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 222\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRNNCell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    223\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    224\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_rnn_get_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgetter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    361\u001b[0m       \u001b[0;31m# Actually call layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 362\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    363\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    364\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    734\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    735\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0min_deferred_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 736\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    737\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    738\u001b[0m           raise ValueError('A layer\\'s `call` method should return a Tensor '\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, state)\u001b[0m\n\u001b[1;32m   1313\u001b[0m                                       [-1, cell.state_size])\n\u001b[1;32m   1314\u001b[0m           \u001b[0mcur_state_pos\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1315\u001b[0;31m         \u001b[0mcur_inp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcur_inp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcur_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1316\u001b[0m         \u001b[0mnew_states\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope, *args, **kwargs)\u001b[0m\n\u001b[1;32m    327\u001b[0m     \u001b[0;31m# method.  See the class docstring for more details.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m     return base_layer.Layer.__call__(self, inputs, state, scope=scope,\n\u001b[0;32m--> 329\u001b[0;31m                                      *args, **kwargs)\n\u001b[0m\u001b[1;32m    330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/layers/base.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    361\u001b[0m       \u001b[0;31m# Actually call layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 362\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLayer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    363\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    364\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    734\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    735\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0min_deferred_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 736\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    737\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    738\u001b[0m           raise ValueError('A layer\\'s `call` method should return a Tensor '\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, state)\u001b[0m\n\u001b[1;32m    626\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    627\u001b[0m     gate_inputs = math_ops.matmul(\n\u001b[0;32m--> 628\u001b[0;31m         array_ops.concat([inputs, h], 1), self._kernel)\n\u001b[0m\u001b[1;32m    629\u001b[0m     \u001b[0mgate_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnn_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias_add\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgate_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_bias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    630\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36mmatmul\u001b[0;34m(a, b, transpose_a, transpose_b, adjoint_a, adjoint_b, a_is_sparse, b_is_sparse, name)\u001b[0m\n\u001b[1;32m   2016\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2017\u001b[0m       return gen_math_ops.mat_mul(\n\u001b[0;32m-> 2018\u001b[0;31m           a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)\n\u001b[0m\u001b[1;32m   2019\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2020\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py\u001b[0m in \u001b[0;36mmat_mul\u001b[0;34m(a, b, transpose_a, transpose_b, name)\u001b[0m\n\u001b[1;32m   4480\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4481\u001b[0m         \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4482\u001b[0;31m       \u001b[0m_six\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tf110/lib/python3.6/site-packages/six.py\u001b[0m in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Matrix size-incompatible: In[0]: [3,10], In[1]: [8,20] [Op:MatMul] name: MultiRNNCell/rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/MatMul/"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('MultiRNNCell') as scope:\n",
    "    # Make rnn\n",
    "    cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    cell = rnn.MultiRNNCell([cell] * 3, state_is_tuple=True) # 3 layers\n",
    "\n",
    "    # rnn in/out\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)\n",
    "    print(\"dynamic rnn: \", outputs)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())  # batch size, unrolling (time), hidden_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dynamic rnn:  tf.Tensor(\n",
      "[[[ 8.7214885e-03  1.0204822e-01 -2.9600459e-01 -6.6370539e-02\n",
      "    8.2512870e-02]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]]\n",
      "\n",
      " [[ 5.1761570e-05  4.5895827e-01 -7.5968915e-01 -8.9063769e-06\n",
      "    1.3682598e-05]\n",
      "  [ 5.6544093e-05  6.8461144e-01 -9.5118338e-01 -9.9540302e-06\n",
      "    2.3344010e-06]\n",
      "  [ 5.5680048e-05  7.8183025e-01 -9.8796153e-01 -1.0060514e-05\n",
      "    3.2277117e-07]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]]\n",
      "\n",
      " [[ 4.9273701e-09  6.1943161e-01 -7.6158565e-01 -1.8325767e-10\n",
      "    4.7917625e-10]\n",
      "  [ 5.5029017e-09  8.4427124e-01 -9.5607680e-01 -2.0636927e-10\n",
      "    8.5343441e-11]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]\n",
      "  [ 0.0000000e+00  0.0000000e+00  0.0000000e+00  0.0000000e+00\n",
      "    0.0000000e+00]]], shape=(3, 5, 5), dtype=float32)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'sess' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-80614b487c16>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"dynamic rnn: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mpp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# batch size, unrolling (time), hidden_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sess' is not defined"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('dynamic_rnn') as scope:\n",
    "    cell = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32,\n",
    "                                         sequence_length=[1, 3, 2])\n",
    "    # lentgh 1 for batch 1, lentgh 2 for batch 2\n",
    "    \n",
    "    print(\"dynamic rnn: \", outputs)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(outputs.eval())  # batch size, unrolling (time), hidden_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sess' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-0869ffc68666>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m                                                       dtype=tf.float32)\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mpp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mpp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sess' is not defined"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('bi-directional') as scope:\n",
    "    # bi-directional rnn\n",
    "    cell_fw = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "    cell_bw = rnn.BasicLSTMCell(num_units=5, state_is_tuple=True)\n",
    "\n",
    "    outputs, states = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, x_data,\n",
    "                                                      sequence_length=[2, 3, 1],\n",
    "                                                      dtype=tf.float32)\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    pp.pprint(sess.run(outputs))\n",
    "    pp.pprint(sess.run(states))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "array([[[  0.,   1.,   2.],\n",
      "        [  3.,   4.,   5.],\n",
      "        [  6.,   7.,   8.],\n",
      "        [  9.,  10.,  11.],\n",
      "        [ 12.,  13.,  14.]],\n",
      "\n",
      "       [[ 15.,  16.,  17.],\n",
      "        [ 18.,  19.,  20.],\n",
      "        [ 21.,  22.,  23.],\n",
      "        [ 24.,  25.,  26.],\n",
      "        [ 27.,  28.,  29.]],\n",
      "\n",
      "       [[ 30.,  31.,  32.],\n",
      "        [ 33.,  34.,  35.],\n",
      "        [ 36.,  37.,  38.],\n",
      "        [ 39.,  40.,  41.],\n",
      "        [ 42.,  43.,  44.]]], dtype=float32)\n",
      "array([[  0.,   1.,   2.],\n",
      "       [  3.,   4.,   5.],\n",
      "       [  6.,   7.,   8.],\n",
      "       [  9.,  10.,  11.],\n",
      "       [ 12.,  13.,  14.],\n",
      "       [ 15.,  16.,  17.],\n",
      "       [ 18.,  19.,  20.],\n",
      "       [ 21.,  22.,  23.],\n",
      "       [ 24.,  25.,  26.],\n",
      "       [ 27.,  28.,  29.],\n",
      "       [ 30.,  31.,  32.],\n",
      "       [ 33.,  34.,  35.],\n",
      "       [ 36.,  37.,  38.],\n",
      "       [ 39.,  40.,  41.],\n",
      "       [ 42.,  43.,  44.]], dtype=float32)\n",
      "array([[[   25.,    28.,    31.,    34.,    37.],\n",
      "        [   70.,    82.,    94.,   106.,   118.],\n",
      "        [  115.,   136.,   157.,   178.,   199.],\n",
      "        [  160.,   190.,   220.,   250.,   280.],\n",
      "        [  205.,   244.,   283.,   322.,   361.]],\n",
      "\n",
      "       [[  250.,   298.,   346.,   394.,   442.],\n",
      "        [  295.,   352.,   409.,   466.,   523.],\n",
      "        [  340.,   406.,   472.,   538.,   604.],\n",
      "        [  385.,   460.,   535.,   610.,   685.],\n",
      "        [  430.,   514.,   598.,   682.,   766.]],\n",
      "\n",
      "       [[  475.,   568.,   661.,   754.,   847.],\n",
      "        [  520.,   622.,   724.,   826.,   928.],\n",
      "        [  565.,   676.,   787.,   898.,  1009.],\n",
      "        [  610.,   730.,   850.,   970.,  1090.],\n",
      "        [  655.,   784.,   913.,  1042.,  1171.]]], dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# flattern based softmax\n",
    "hidden_size=3\n",
    "sequence_length=5\n",
    "batch_size=3\n",
    "num_classes=5\n",
    "\n",
    "pp.pprint(x_data) # hidden_size=3, sequence_length=4, batch_size=2\n",
    "x_data = x_data.reshape(-1, hidden_size)\n",
    "pp.pprint(x_data)\n",
    "\n",
    "softmax_w = np.arange(15, dtype=np.float32).reshape(hidden_size, num_classes)\n",
    "outputs = np.matmul(x_data, softmax_w)\n",
    "outputs = outputs.reshape(-1, sequence_length, num_classes) # batch, seq, class\n",
    "pp.pprint(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:  0.596759\n"
     ]
    }
   ],
   "source": [
    "# [batch_size, sequence_length]\n",
    "y_data = tf.constant([[1, 1, 1]])\n",
    "\n",
    "# [batch_size, sequence_length, emb_dim ]\n",
    "prediction = tf.constant([[[0.2, 0.7], [0.6, 0.2], [0.2, 0.9]]], dtype=tf.float32)\n",
    "\n",
    "# [batch_size * sequence_length]\n",
    "weights = tf.constant([[1, 1, 1]], dtype=tf.float32)\n",
    "\n",
    "sequence_loss = tf.contrib.seq2seq.sequence_loss(logits=prediction, targets=y_data, weights=weights)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "print(\"Loss: \", sequence_loss.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss1:  0.513015 Loss2:  0.371101 Loss3:  1.31326 Loss4:  1.31326\n"
     ]
    }
   ],
   "source": [
    "# [batch_size, sequence_length]\n",
    "y_data = tf.constant([[1, 1, 1]])\n",
    "\n",
    "# [batch_size, sequence_length, emb_dim ]\n",
    "prediction1 = tf.constant([[[0.3, 0.7], [0.3, 0.7], [0.3, 0.7]]], dtype=tf.float32)\n",
    "prediction2 = tf.constant([[[0.1, 0.9], [0.1, 0.9], [0.1, 0.9]]], dtype=tf.float32)\n",
    "\n",
    "prediction3 = tf.constant([[[1, 0], [1, 0], [1, 0]]], dtype=tf.float32)\n",
    "prediction4 = tf.constant([[[0, 1], [1, 0], [0, 1]]], dtype=tf.float32)\n",
    "\n",
    "# [batch_size * sequence_length]\n",
    "weights = tf.constant([[1, 1, 1]], dtype=tf.float32)\n",
    "\n",
    "sequence_loss1 = tf.contrib.seq2seq.sequence_loss(prediction1, y_data, weights)\n",
    "sequence_loss2 = tf.contrib.seq2seq.sequence_loss(prediction2, y_data, weights)\n",
    "sequence_loss3 = tf.contrib.seq2seq.sequence_loss(prediction3, y_data, weights)\n",
    "sequence_loss4 = tf.contrib.seq2seq.sequence_loss(prediction3, y_data, weights)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "print(\"Loss1: \", sequence_loss1.eval(),\n",
    "      \"Loss2: \", sequence_loss2.eval(),\n",
    "      \"Loss3: \", sequence_loss3.eval(),\n",
    "      \"Loss4: \", sequence_loss4.eval())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}