2-recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (73401, 32, 32, 1) (73401, 6)\n",
      "Validation set (10000, 32, 32, 1) (10000, 6)\n",
      "Test set (13068, 32, 32, 1) (13068, 6)\n"
     ]
    }
   ],
   "source": [
    "pickle_file = 'SVHN.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_data']\n",
    "    train_labels = save['train_labels']\n",
    "    test_dataset = save['test_data']\n",
    "    test_labels = save['test_labels']\n",
    "    test_filenames = save['test_filenames']\n",
    "    valid_dataset = save['valid_data']\n",
    "    valid_labels = save['valid_labels']\n",
    "    del save\n",
    "    \n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gaussian_filter(kernel_shape):\n",
    "    x = np.zeros(kernel_shape, dtype = float)\n",
    "    mid = np.floor(kernel_shape[0] / 2.)\n",
    "    \n",
    "    for kernel_idx in xrange(0, kernel_shape[2]):\n",
    "        for i in xrange(0, kernel_shape[0]):\n",
    "            for j in xrange(0, kernel_shape[1]):\n",
    "                x[i, j, kernel_idx, 0] = gauss(i - mid, j - mid)\n",
    "    \n",
    "    return tf.convert_to_tensor(x / np.sum(x), dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gauss(x, y, sigma=3.0):\n",
    "    Z = 2 * np.pi * sigma ** 2\n",
    "    return  1. / Z * np.exp(-(x ** 2 + y ** 2) / (2. * sigma ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def LecunLCN(X, image_shape, threshold=1e-4, radius=7, use_divisor=True):\n",
    "    \"\"\"Local Contrast Normalization\"\"\"\n",
    "    \"\"\"[http://yann.lecun.com/exdb/publis/pdf/jarrett-iccv-09.pdf]\"\"\"\n",
    "\n",
    "    # Get Gaussian filter\n",
    "    filter_shape = (radius, radius, image_shape[3], 1)\n",
    "\n",
    "    #self.filters = theano.shared(self.gaussian_filter(filter_shape), borrow=True)\n",
    "    filters = gaussian_filter(filter_shape)\n",
    "    X = tf.convert_to_tensor(X, dtype=tf.float32)\n",
    "    # Compute the Guassian weighted average by means of convolution\n",
    "    convout = tf.nn.conv2d(X, filters, [1,1,1,1], 'SAME')\n",
    "\n",
    "    # Subtractive step\n",
    "    mid = int(np.floor(filter_shape[1] / 2.))\n",
    "\n",
    "    # Make filter dimension broadcastable and subtract\n",
    "    centered_X = tf.sub(X, convout)\n",
    "\n",
    "    # Boolean marks whether or not to perform divisive step\n",
    "    if use_divisor:\n",
    "        # Note that the local variances can be computed by using the centered_X\n",
    "        # tensor. If we convolve this with the mean filter, that should give us\n",
    "        # the variance at each point. We simply take the square root to get our\n",
    "        # denominator\n",
    "\n",
    "        # Compute variances\n",
    "        sum_sqr_XX = tf.nn.conv2d(tf.square(centered_X), filters, [1,1,1,1], 'SAME')\n",
    "\n",
    "        # Take square root to get local standard deviation\n",
    "        denom = tf.sqrt(sum_sqr_XX)\n",
    "\n",
    "        per_img_mean = tf.reduce_mean(denom)\n",
    "        divisor = tf.maximum(per_img_mean, denom)\n",
    "        # Divisise step\n",
    "        new_X = tf.truediv(centered_X, tf.maximum(divisor, threshold))\n",
    "    else:\n",
    "        new_X = centered_X\n",
    "    return new_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels, printstat=False):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 2).T == labels) / predictions.shape[1] / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "image_size = 32\n",
    "num_labels = 11 # 0-9, + blank \n",
    "num_channels = 1 # grayscale\n",
    "\n",
    "batch_size = 64\n",
    "patch_size = 5\n",
    "depth1 = 32\n",
    "depth2 = 64\n",
    "depth3 = 128\n",
    "num_hidden1 = 512\n",
    "num_hidden2 = 128\n",
    "num_hidden3 = 32\n",
    "shape = [batch_size, image_size, image_size, num_channels]\n",
    "\n",
    "# Construct a 7-layer CNN.\n",
    "# C1: convolutional layer, batch_size x 32 x 32 x 11, filter size: 5 x 5 x 1 x 32\n",
    "# P2: pooling layer, batch_size x 14 x 14 x 16\n",
    "# C3: convolutional layer, batch_size x 10 x 10 x 32, filter size: 5 x 5 x 16 x 64\n",
    "# P4: pooling layer, batch_size x 5 x 5 x 32\n",
    "# C5: convolutional layer, batch_size x 1 x 1 x 64, filter size: 5 x 5 x 32 x 128\n",
    "# Dropout\n",
    "# F6: fully-connected layer, weight size: 64 x 11\n",
    "# Output layer, weight size: 16 x 10\n",
    "\n",
    "train_size = train_labels.shape[0]\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data placeholders.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))\n",
    "    tf_train_labels = tf.placeholder(tf.int32, shape=(batch_size, 6))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "    # Variables.\n",
    "    layer1_weights = tf.get_variable(\"W1\", shape=[patch_size, patch_size, num_channels, depth1],\\\n",
    "                                     initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    layer1_biases = tf.Variable(tf.constant(1.0, shape=[depth1]), name='B1')\n",
    "    \n",
    "    layer2_weights = tf.get_variable(\"W2\", shape=[patch_size, patch_size, depth1, depth2],\\\n",
    "                                     initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth2]), name='B2')\n",
    "    \n",
    "    layer3_weights = tf.get_variable(\"W3\", shape=[patch_size, patch_size, depth2, num_hidden1],\\\n",
    "                                     initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden1]), name='B3')\n",
    "\n",
    "    s1_w1 = tf.get_variable(\"WS11\", shape=[num_hidden1, num_hidden2],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s1_b1 = tf.Variable(tf.constant(1.0, shape=[num_hidden2]), name='BS11')\n",
    "    \n",
    "    s2_w1 = tf.get_variable(\"WS12\", shape=[num_hidden1, num_hidden2],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s2_b1 = tf.Variable(tf.constant(1.0, shape=[num_hidden2]), name='BS12')\n",
    "    \n",
    "    s3_w1 = tf.get_variable(\"WS13\", shape=[num_hidden1, num_hidden2],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s3_b1 = tf.Variable(tf.constant(1.0, shape=[num_hidden2]), name='BS13')\n",
    "    \n",
    "    s4_w1 = tf.get_variable(\"WS14\", shape=[num_hidden1, num_hidden2],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s4_b1 = tf.Variable(tf.constant(1.0, shape=[num_hidden2]), name='BS14')\n",
    "    \n",
    "    s5_w1 = tf.get_variable(\"WS15\", shape=[num_hidden1, num_hidden2],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s5_b1 = tf.Variable(tf.constant(1.0, shape=[num_hidden2]), name='BS15')\n",
    "    \n",
    "\n",
    "    s1_w2 = tf.get_variable(\"WS21\", shape=[num_hidden2, num_hidden3],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s1_b2 = tf.Variable(tf.constant(1.0, shape=[num_hidden3]), name='BS21')\n",
    "    \n",
    "    s2_w2 = tf.get_variable(\"WS22\", shape=[num_hidden2, num_hidden3],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s2_b2 = tf.Variable(tf.constant(1.0, shape=[num_hidden3]), name='BS22')\n",
    "    \n",
    "    s3_w2 = tf.get_variable(\"WS23\", shape=[num_hidden2, num_hidden3],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s3_b2 = tf.Variable(tf.constant(1.0, shape=[num_hidden3]), name='BS23')\n",
    "    \n",
    "    s4_w2 = tf.get_variable(\"WS24\", shape=[num_hidden2, num_hidden3],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s4_b2 = tf.Variable(tf.constant(1.0, shape=[num_hidden3]), name='BS24')\n",
    "    \n",
    "    s5_w2 = tf.get_variable(\"WS25\", shape=[num_hidden2, num_hidden3],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s5_b2 = tf.Variable(tf.constant(1.0, shape=[num_hidden3]), name='BS25')\n",
    "    \n",
    "\n",
    "    s1_w3 = tf.get_variable(\"WS31\", shape=[num_hidden3, num_labels],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s1_b3 = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS31')\n",
    "    \n",
    "    s2_w3 = tf.get_variable(\"WS32\", shape=[num_hidden3, num_labels],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s2_b3 = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS32')\n",
    "    \n",
    "    s3_w3 = tf.get_variable(\"WS33\", shape=[num_hidden3, num_labels],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s3_b3 = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS33')\n",
    "    \n",
    "    s4_w3 = tf.get_variable(\"WS34\", shape=[num_hidden3, num_labels],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s4_b3 = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS34')\n",
    "    \n",
    "    s5_w3 = tf.get_variable(\"WS35\", shape=[num_hidden3, num_labels],\\\n",
    "                           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s5_b3 = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS35')\n",
    "\n",
    "    # Model.\n",
    "    def model(data, keep_prob, shape):\n",
    "        LCN = LecunLCN(data, shape)\n",
    "        \n",
    "        conv = tf.nn.conv2d(LCN, layer1_weights, [1,1,1,1], 'VALID', name='C1')\n",
    "        hidden = tf.nn.relu(conv + layer1_biases)\n",
    "        lrn = tf.nn.local_response_normalization(hidden)\n",
    "        sub = tf.nn.max_pool(lrn, [1,2,2,1], [1,2,2,1], 'SAME', name='P2')\n",
    "        \n",
    "        conv = tf.nn.conv2d(sub, layer2_weights, [1,1,1,1], padding='VALID', name='C3')\n",
    "        hidden = tf.nn.relu(conv + layer2_biases)\n",
    "        lrn = tf.nn.local_response_normalization(hidden)\n",
    "        sub = tf.nn.max_pool(lrn, [1,2,2,1], [1,2,2,1], 'SAME', name='P4')\n",
    "        \n",
    "        conv = tf.nn.conv2d(sub, layer3_weights, [1,1,1,1], padding='VALID', name='C5')        \n",
    "        hidden = tf.nn.relu(conv + layer3_biases)\n",
    "        hidden = tf.nn.dropout(hidden, keep_prob)\n",
    "        \n",
    "        shape = hidden.get_shape().as_list()\n",
    "        reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "        #print(reshape.get_shape())\n",
    "        # hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)\n",
    "        hidden11 = tf.nn.relu(tf.matmul(reshape, s1_w1) + s1_b1)\n",
    "        hidden21 = tf.nn.relu(tf.matmul(reshape, s2_w1) + s2_b1)\n",
    "        hidden31 = tf.nn.relu(tf.matmul(reshape, s3_w1) + s3_b1)\n",
    "        hidden41 = tf.nn.relu(tf.matmul(reshape, s4_w1) + s4_b1)\n",
    "        hidden51 = tf.nn.relu(tf.matmul(reshape, s5_w1) + s5_b1)\n",
    "        \n",
    "        hidden12 = tf.nn.relu(tf.matmul(hidden11, s1_w2) + s1_b2)\n",
    "        hidden22 = tf.nn.relu(tf.matmul(hidden21, s2_w2) + s2_b2)\n",
    "        hidden32 = tf.nn.relu(tf.matmul(hidden31, s3_w2) + s3_b2)\n",
    "        hidden42 = tf.nn.relu(tf.matmul(hidden41, s4_w2) + s4_b2)\n",
    "        hidden52 = tf.nn.relu(tf.matmul(hidden51, s5_w2) + s5_b2)\n",
    "        \n",
    "        logits1 = tf.matmul(hidden12, s1_w3) + s1_b3\n",
    "        logits2 = tf.matmul(hidden22, s2_w3) + s2_b3\n",
    "        logits3 = tf.matmul(hidden32, s3_w3) + s3_b3\n",
    "        logits4 = tf.matmul(hidden42, s4_w3) + s4_b3\n",
    "        logits5 = tf.matmul(hidden52, s5_w3) + s5_b3\n",
    "        \n",
    "        return [logits1, logits2, logits3, logits4, logits5]\n",
    "\n",
    "    # Training computation.\n",
    "    [logits1, logits2, logits3, logits4, logits5] = model(tf_train_dataset, 0.6, shape)\n",
    "    \n",
    "    # L2 regularization for the fully connected parameters.\n",
    "    regularizers = [(tf.nn.l2_loss(s1_w2) + tf.nn.l2_loss(s1_b2)),\n",
    "                    (tf.nn.l2_loss(s2_w2) + tf.nn.l2_loss(s2_b2)),\n",
    "                    (tf.nn.l2_loss(s3_w2) + tf.nn.l2_loss(s3_b2)),\n",
    "                    (tf.nn.l2_loss(s4_w2) + tf.nn.l2_loss(s4_b2)),\n",
    "                    (tf.nn.l2_loss(s5_w2) + tf.nn.l2_loss(s5_b2)),]\n",
    "    \n",
    "    post_logits = [tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits1, tf_train_labels[:,1])),\n",
    "                   tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits2, tf_train_labels[:,2])),\n",
    "                   tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits3, tf_train_labels[:,3])),\n",
    "                   tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits4, tf_train_labels[:,4])),\n",
    "                   tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits5, tf_train_labels[:,5]))]\n",
    "    \n",
    "    loss = (post_logits[0] + 4e-4 * regularizers[0]) + (post_logits[1] + 4e-4 * regularizers[1]) + \\\n",
    "           (post_logits[2] + 4e-4 * regularizers[2]) + (post_logits[3] + 4e-4 * regularizers[3]) + \\\n",
    "           (post_logits[4] + 4e-4 * regularizers[4])\n",
    "    \n",
    "    # Optimizer.\n",
    "    batch = tf.Variable(0, dtype=tf.float32)\n",
    "    \n",
    "    # Decay once per epoch, using an exponential schedule starting at 0.01.\n",
    "    learning_rate = tf.train.exponential_decay(0.01, batch * batch_size, train_size, 0.95, staircase=True)\n",
    "    \n",
    "    # Use simple momentum for the optimization.\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, 0.9).minimize(loss, global_step=batch)\n",
    "    \n",
    "    # Predictions for the training, validation and test data.\n",
    "    train_logits = model(tf_train_dataset, 1.0, shape)\n",
    "    train_prediction = tf.pack([tf.nn.softmax(train_logits[0]), tf.nn.softmax(train_logits[1]),\\\n",
    "                                tf.nn.softmax(train_logits[2]), tf.nn.softmax(train_logits[3]),\\\n",
    "                                tf.nn.softmax(train_logits[4])])\n",
    "    \n",
    "    valid_logits = model(tf_valid_dataset, 1.0, shape)\n",
    "    valid_prediction = tf.pack([tf.nn.softmax(valid_logits[0]), tf.nn.softmax(valid_logits[1]),\\\n",
    "                                tf.nn.softmax(valid_logits[2]), tf.nn.softmax(valid_logits[3]),\\\n",
    "                                tf.nn.softmax(valid_logits[4])])\n",
    "    \n",
    "    test_logits = model(tf_test_dataset, 1.0, shape)\n",
    "    test_prediction = tf.pack([tf.nn.softmax(test_logits[0]), tf.nn.softmax(test_logits[1]),\\\n",
    "                               tf.nn.softmax(test_logits[2]), tf.nn.softmax(test_logits[3]),\\\n",
    "                               tf.nn.softmax(test_logits[4])])\n",
    "\n",
    "    saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 21.470789\n",
      "Minibatch accuracy: 21.6%\n",
      "Validation accuracy: 52.1%\n",
      "Minibatch loss at step 1000: 6.032491\n",
      "Minibatch accuracy: 61.6%\n",
      "Validation accuracy: 58.5%\n",
      "Minibatch loss at step 2000: 2.941924\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 3000: 1.670177\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 89.1%\n",
      "Minibatch loss at step 4000: 1.864727\n",
      "Minibatch accuracy: 90.9%\n",
      "Validation accuracy: 91.5%\n",
      "Minibatch loss at step 5000: 0.914149\n",
      "Minibatch accuracy: 97.5%\n",
      "Validation accuracy: 92.8%\n",
      "Minibatch loss at step 6000: 1.433883\n",
      "Minibatch accuracy: 94.7%\n",
      "Validation accuracy: 93.5%\n",
      "Minibatch loss at step 7000: 1.208452\n",
      "Minibatch accuracy: 95.9%\n",
      "Validation accuracy: 94.4%\n",
      "Minibatch loss at step 8000: 1.144454\n",
      "Minibatch accuracy: 95.9%\n",
      "Validation accuracy: 94.7%\n",
      "Minibatch loss at step 9000: 0.647428\n",
      "Minibatch accuracy: 97.5%\n",
      "Validation accuracy: 94.8%\n",
      "Minibatch loss at step 10000: 0.704464\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 95.1%\n",
      "Minibatch loss at step 11000: 0.534398\n",
      "Minibatch accuracy: 98.1%\n",
      "Validation accuracy: 94.9%\n",
      "Minibatch loss at step 12000: 0.493030\n",
      "Minibatch accuracy: 99.4%\n",
      "Validation accuracy: 95.0%\n",
      "Minibatch loss at step 13000: 0.743866\n",
      "Minibatch accuracy: 98.1%\n",
      "Validation accuracy: 95.3%\n",
      "Minibatch loss at step 14000: 0.688242\n",
      "Minibatch accuracy: 97.2%\n",
      "Validation accuracy: 95.4%\n",
      "Minibatch loss at step 15000: 0.474407\n",
      "Minibatch accuracy: 99.1%\n",
      "Validation accuracy: 95.8%\n",
      "Minibatch loss at step 16000: 0.653862\n",
      "Minibatch accuracy: 98.8%\n",
      "Validation accuracy: 95.6%\n",
      "Minibatch loss at step 17000: 0.818901\n",
      "Minibatch accuracy: 98.8%\n",
      "Validation accuracy: 95.6%\n",
      "Minibatch loss at step 18000: 0.526244\n",
      "Minibatch accuracy: 98.1%\n",
      "Validation accuracy: 95.7%\n",
      "Minibatch loss at step 19000: 0.321311\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 95.7%\n",
      "Minibatch loss at step 20000: 0.555234\n",
      "Minibatch accuracy: 99.7%\n",
      "Validation accuracy: 95.6%\n",
      "Minibatch loss at step 21000: 0.350318\n",
      "Minibatch accuracy: 99.4%\n",
      "Validation accuracy: 95.7%\n",
      "Minibatch loss at step 22000: 0.309962\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 95.8%\n",
      "Minibatch loss at step 23000: 0.484072\n",
      "Minibatch accuracy: 98.4%\n",
      "Validation accuracy: 95.8%\n",
      "Minibatch loss at step 24000: 0.195024\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 95.8%\n",
      "Minibatch loss at step 25000: 0.220138\n",
      "Minibatch accuracy: 99.7%\n",
      "Validation accuracy: 95.9%\n",
      "Test accuracy: 94.3%\n",
      "Model saved in file: cnn.convnet\n"
     ]
    }
   ],
   "source": [
    "num_steps = 25001\n",
    "\n",
    "loss_values = []\n",
    "train_accuracies = []\n",
    "validation_accuracies = []\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.initialize_all_variables().run()  \n",
    "    print('Initialized')\n",
    "    \n",
    "    for step in range(num_steps):\n",
    "        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
    "        batch_labels = train_labels[offset:(offset + batch_size),:]\n",
    "        feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    \n",
    "        #if step % 10 == 0:\n",
    "        #    loss_values.append(l)\n",
    "            \n",
    "        if (step % 1000 == 0): \n",
    "            train_accuracy = accuracy(predictions, batch_labels[:,1:6])\n",
    "            valid_accuracy = accuracy(valid_prediction.eval(), valid_labels[:,1:6])\n",
    "            print('Minibatch loss at step %d: %f' % (step, l))\n",
    "            print('Minibatch accuracy: %.1f%%' % train_accuracy)    \n",
    "            print('Validation accuracy: %.1f%%' % valid_accuracy)\n",
    "            train_accuracies.append(train_accuracy)\n",
    "            validation_accuracies.append(valid_accuracy)\n",
    "            \n",
    "    \n",
    "    print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels[:,1:6]))\n",
    "    \n",
    "    save_path = saver.save(session, \"cnn.convnet\")\n",
    "    print(\"Model saved in file: %s\" % save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.figure(1)\n",
    "\n",
    "plt.bar(np.arange(0,10001,10), loss_values, align='center')\n",
    "plt.title('Dropout = 0.9375')\n",
    "plt.xlabel('Training Step')\n",
    "plt.yticks(np.arange(0,21,5))\n",
    "plt.xticks(np.arange(0,10500,5000))\n",
    "plt.ylabel('Loss Value')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.figure(2)\n",
    "\n",
    "plt.plot(np.arange(0,10001,1000), train_accuracies, label = 'Training Accuracy')\n",
    "plt.plot(np.arange(0,10001,1000), validation_accuracies, label = 'Validation Accuracy')\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "plt.title('Accuracy improvement on training phase')\n",
    "plt.xlabel('Training Step')\n",
    "plt.ylabel('Accuracy, %')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import os\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (20.0, 20.0)\n",
    "f, ax = plt.subplots(nrows=1, ncols=10)\n",
    "\n",
    "im_samples = []\n",
    "for i, item in enumerate(np.sort(np.random.randint(0, test_labels.shape[0], size=10))):\n",
    "    filename = test_filenames[item]\n",
    "    fullname = os.path.join('test', filename)\n",
    "    im = Image.open(fullname)\n",
    "    house_num = ''\n",
    "    for k in np.arange(test_labels[item,0]):\n",
    "        house_num += str(test_labels[item,k+1])\n",
    "        \n",
    "    im_samples.extend([item])\n",
    "    ax[i].axis('off')\n",
    "    ax[i].set_title(house_num, loc='center')\n",
    "    ax[i].imshow(im)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data.\n",
    "    tf_test_dataset = tf.placeholder(tf.float32, shape=(10, 32, 32, 1))\n",
    "\n",
    "    # Variables.\n",
    "    layer1_weights = tf.get_variable(\"W1\", shape=[patch_size, patch_size, num_channels, depth1],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    layer1_biases = tf.Variable(tf.constant(1.0, shape=[depth1]), name='B1')\n",
    "    layer2_weights = tf.get_variable(\"W2\", shape=[patch_size, patch_size, depth1, depth2],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth2]), name='B2')\n",
    "    layer3_weights = tf.get_variable(\"W3\", shape=[patch_size, patch_size, depth2, num_hidden1],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden1]), name='B3')\n",
    "\n",
    "    s1_w = tf.get_variable(\"WS1\", shape=[num_hidden1, num_labels],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s1_b = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS1')\n",
    "    s2_w = tf.get_variable(\"WS2\", shape=[num_hidden1, num_labels],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s2_b = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS2')\n",
    "    s3_w = tf.get_variable(\"WS3\", shape=[num_hidden1, num_labels],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s3_b = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS3')\n",
    "    s4_w = tf.get_variable(\"WS4\", shape=[num_hidden1, num_labels],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s4_b = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS4')\n",
    "    s5_w = tf.get_variable(\"WS5\", shape=[num_hidden1, num_labels],\\\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    s5_b = tf.Variable(tf.constant(1.0, shape=[num_labels]), name='BS5')\n",
    "  \n",
    "    # Model.\n",
    "    def model(data, keep_prob, shape):\n",
    "        LCN = LecunLCN(data, shape)\n",
    "        conv = tf.nn.conv2d(LCN, layer1_weights, [1,1,1,1], 'VALID', name='C1')\n",
    "        hidden = tf.nn.relu(conv + layer1_biases)\n",
    "        lrn = tf.nn.local_response_normalization(hidden)\n",
    "        sub = tf.nn.max_pool(lrn, [1,2,2,1], [1,2,2,1], 'SAME', name='S2')\n",
    "        conv = tf.nn.conv2d(sub, layer2_weights, [1,1,1,1], padding='VALID', name='C3')\n",
    "        hidden = tf.nn.relu(conv + layer2_biases)\n",
    "        lrn = tf.nn.local_response_normalization(hidden)\n",
    "        sub = tf.nn.max_pool(lrn, [1,2,2,1], [1,2,2,1], 'SAME', name='S4')\n",
    "        conv = tf.nn.conv2d(sub, layer3_weights, [1,1,1,1], padding='VALID', name='C5')\n",
    "        hidden = tf.nn.relu(conv + layer3_biases)\n",
    "        hidden = tf.nn.dropout(hidden, keep_prob)\n",
    "        shape = hidden.get_shape().as_list()\n",
    "        reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])\n",
    "        logits1 = tf.matmul(reshape, s1_w) + s1_b\n",
    "        logits2 = tf.matmul(reshape, s2_w) + s2_b\n",
    "        logits3 = tf.matmul(reshape, s3_w) + s3_b\n",
    "        logits4 = tf.matmul(reshape, s4_w) + s4_b\n",
    "        logits5 = tf.matmul(reshape, s5_w) + s5_b\n",
    "        return [logits1, logits2, logits3, logits4, logits5]\n",
    "  \n",
    "    # Training computation.\n",
    "    [logits1, logits2, logits3, logits4, logits5] = model(tf_test_dataset, 1, [10, 32, 32, 1])\n",
    "\n",
    "    predict = tf.pack([tf.nn.softmax(logits1),tf.nn.softmax(logits2),tf.nn.softmax(logits3),\\\n",
    "                       tf.nn.softmax(logits4),tf.nn.softmax(logits5)])\n",
    "    test_prediction = tf.transpose(tf.argmax(predict, 2))\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    saver.restore(session, \"cnn.convnet\")\n",
    "    print(\"Model restored.\")  \n",
    "\n",
    "    print('Initialized')\n",
    "    test_prediction = session.run(test_prediction, feed_dict={tf_test_dataset : test_dataset[im_samples,:,:,:],})\n",
    "    print(test_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (15.0, 15.0)\n",
    "f, ax = plt.subplots(nrows=1, ncols=10)\n",
    "\n",
    "for i, j in enumerate(im_samples):\n",
    "    filename = test_filenames[j]\n",
    "    fullname = os.path.join('test', filename)\n",
    "    im = Image.open(fullname)\n",
    "    house_num = ''\n",
    "    for k in np.arange(np.sum(test_prediction[i,:] != 10)):\n",
    "        house_num += str(test_prediction[i,k])\n",
    "\n",
    "    ax[i].axis('off')\n",
    "    ax[i].set_title(house_num, loc='center')\n",
    "    ax[i].imshow(im)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}