Keras-model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting deeper with Keras\n",
    "* Tensorflow is a powerful and flexible tool, but coding large neural architectures with it is tedious.\n",
    "* There are plenty of deep learning toolkits that work on top of it like Slim, TFLearn, Sonnet, Keras.\n",
    "* Choice is matter of taste and particular task\n",
    "* We'll be using Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"..\")\n",
    "import grading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use preloaded keras datasets and models\n",
    "! mkdir -p ~/.keras/datasets\n",
    "! mkdir -p ~/.keras/models\n",
    "! ln -s $(realpath ../readonly/keras/datasets/*) ~/.keras/datasets/\n",
    "! ln -s $(realpath ../readonly/keras/models/*) ~/.keras/models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from preprocessed_mnist import load_dataset\n",
    "import keras\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = load_dataset()\n",
    "y_train,y_val,y_test = map(keras.utils.np_utils.to_categorical,[y_train,y_val,y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADn9JREFUeJzt3X9sXfV5x/HPU8dxlhDauCmeSzMSIC3QsIbtKoCIgImR\npQgpoKqhUVWljDVdC3RsmQTLpjWb2JRNLVXKGJJZsyQVv0oLIn+wVmBV0GrgYbIQfpVfwV0TjE1w\nIYHSxLGf/eGTygXf73XuPfeeaz/vl2T53vOcc8+jk3x87r3fe8/X3F0A4vlA0Q0AKAbhB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgCD8Q1IxG7mymtfkszWnkLoFQfq13dNgP2WTWrSn8ZrZS0mZJLZL+\nw903pdafpTk62y6qZZcAEnq8e9LrVv2038xaJN0i6dOSzpC0xszOqPbxADRWLa/5l0l6yd33uPth\nSXdJWpVPWwDqrZbwnyjpF+Pu782W/RYzW2dmvWbWO6xDNewOQJ7q/m6/u3e5e8ndS61qq/fuAExS\nLeHfJ2nBuPsfy5YBmAJqCf/jkhab2SIzmynpc5J25NMWgHqreqjP3Y+Y2TWSfqSxob4t7v5Mbp0B\nqKuaxvnd/QFJD+TUC4AG4uO9QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ER\nfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANB\nEX4gKMIPBFXTLL1m1ifpoKQRSUfcvZRHU8iPzUj/E7d8ZH5d9//8Xy8sWxuZPZrc9qRTBpP12V+1\nZP21m2aWre0s3Z3cdv/IO8n62fesT9ZP/avHkvVmUFP4M3/k7vtzeBwADcTTfiCoWsPvkh4ysyfM\nbF0eDQFojFqf9i93931mdoKkB83sZ+7+yPgVsj8K6yRplmbXuDsAeanpzO/u+7Lfg5Luk7RsgnW6\n3L3k7qVWtdWyOwA5qjr8ZjbHzOYevS1phaSn82oMQH3V8rS/Q9J9Znb0ce5w9x/m0hWAuqs6/O6+\nR9Kncuxl2mo5fXGy7m2tyfqrF3woWX/3nPJj0u0fTI9X/+RT6fHuIv3Xr+Ym6//ybyuT9Z4z7yhb\ne2X43eS2mwYuTtY/+hNP1qcChvqAoAg/EBThB4Ii/EBQhB8IivADQeXxrb7wRi78g2T9pq23JOsf\nby3/1dPpbNhHkvW/v/mLyfqMd9LDbefec03Z2tx9R5Lbtu1PDwXO7u1J1qcCzvxAUIQfCIrwA0ER\nfiAowg8ERfiBoAg/EBTj/Dloe/7VZP2JXy9I1j/eOpBnO7la339Osr7n7fSlv7ee8v2ytbdG0+P0\nHd/+72S9nqb+F3Yr48wPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0GZe+NGNI+3dj/bLmrY/prF0JXn\nJusHVqYvr92y+7hk/cmv3nzMPR114/7fT9YfvyA9jj/y5lvJup9b/urufV9LbqpFa55Mr4D36fFu\nHfCh9NzlGc78QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxXF+M9si6VJJg+6+JFvWLuluSQsl9Ula\n7e6/rLSzqOP8lbTM/3CyPvLGULL+yh3lx+qfOX9Lcttl/3xtsn7CLcV9px7HLu9x/q2S3jsR+g2S\nut19saTu7D6AKaRi+N39EUnvPfWskrQtu71N0mU59wWgzqp9zd/h7v3Z7dckdeTUD4AGqfkNPx97\n06DsGwdmts7Mes2sd1iHat0dgJxUG/4BM+uUpOz3YLkV3b3L3UvuXmpVW5W7A5C3asO/Q9La7PZa\nSffn0w6ARqkYfjO7U9Kjkj5hZnvN7CpJmyRdbGYvSvrj7D6AKaTidfvdfU2ZEgP2ORnZ/0ZN2w8f\nmFn1tp/8/LPJ+uu3tqQfYHSk6n2jWHzCDwiK8ANBEX4gKMIPBEX4gaAIPxAUU3RPA6df/0LZ2pVn\npkdk//Ok7mT9gs9enazPvfuxZB3NizM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwTFOP80kJom+42v\nnJ7c9v92vJus33Dj9mT9b1Zfnqz7/36wbG3BPz2a3FYNnD4+Is78QFCEHwiK8ANBEX4gKMIPBEX4\ngaAIPxBUxSm688QU3c1n6E/PTdZv//o3kvVFM2ZVve9Pbr8mWV98W3+yfmRPX9X7nq7ynqIbwDRE\n+IGgCD8QFOEHgiL8QFCEHwiK8ANBVRznN7Mtki6VNOjuS7JlGyV9SdLr2Wob3P2BSjtjnH/q8fOW\nJuvHb9qbrN958o+q3vdpP/6zZP0T/1D+OgaSNPLinqr3PVXlPc6/VdLKCZZ/y92XZj8Vgw+guVQM\nv7s/ImmoAb0AaKBaXvNfa2a7zWyLmc3LrSMADVFt+G+VdLKkpZL6JX2z3Ipmts7Mes2sd1iHqtwd\ngLxVFX53H3D3EXcflXSbpGWJdbvcveTupVa1VdsngJxVFX4z6xx393JJT+fTDoBGqXjpbjO7U9KF\nkuab2V5JX5d0oZktleSS+iR9uY49AqgDvs+PmrR0nJCsv3rFqWVrPddvTm77gQpPTD//yopk/a3l\nbyTr0xHf5wdQEeEHgiL8QFCEHwiK8ANBEX4gKIb6UJjv7U1P0T3bZibrv/LDyfql115X/rHv60lu\nO1Ux1AegIsIPBEX4gaAIPxAU4QeCIvxAUIQfCKri9/kR2+jy9KW7X/5seoruJUv7ytYqjeNXcvPQ\nWcn67Pt7a3r86Y4zPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExTj/NGelJcn6C19Lj7Xfdt62ZP38\nWenv1NfikA8n648NLUo/wGh/jt1MP5z5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCoiuP8ZrZA0nZJ\nHZJcUpe7bzazdkl3S1ooqU/Sanf/Zf1ajWvGopOS9Zev/GjZ2sYr7kpu+5nj9lfVUx42DJSS9Yc3\nn5Osz9uWvu4/0iZz5j8iab27nyHpHElXm9kZkm6Q1O3uiyV1Z/cBTBEVw+/u/e6+M7t9UNJzkk6U\ntErS0Y9/bZN0Wb2aBJC/Y3rNb2YLJZ0lqUdSh7sf/fzkaxp7WQBgiph0+M3sOEk/kHSdux8YX/Ox\nCf8mnPTPzNaZWa+Z9Q7rUE3NAsjPpMJvZq0aC/7t7n5vtnjAzDqzeqekwYm2dfcudy+5e6lVbXn0\nDCAHFcNvZibpO5Kec/ebxpV2SFqb3V4r6f782wNQL5P5Su95kr4g6Skz25Ut2yBpk6TvmdlVkn4u\naXV9Wpz6Ziz8vWT9rT/sTNav+McfJut//qF7k/V6Wt+fHo579N/LD+e1b/2f5LbzRhnKq6eK4Xf3\nn0oqN9/3Rfm2A6BR+IQfEBThB4Ii/EBQhB8IivADQRF+ICgu3T1JMzp/t2xtaMuc5LZfWfRwsr5m\n7kBVPeXhmn3Lk/Wdt6an6J7//aeT9faDjNU3K878QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBUmHH+\nw3+Svkz04b8cStY3nPpA2dqK33mnqp7yMjDybtna+TvWJ7c97e9+lqy3v5kepx9NVtHMOPMDQRF+\nICjCDwRF+IGgCD8QFOEHgiL8QFBhxvn7Lkv/nXvhzHvqtu9b3jwlWd/88Ipk3UbKXTl9zGk3vlK2\ntnigJ7ntSLKK6YwzPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EZe6eXsFsgaTtkjokuaQud99sZhsl\nfUnS69mqG9y9/JfeJR1v7X62Mas3UC893q0DPpT+YEhmMh/yOSJpvbvvNLO5kp4wswez2rfc/RvV\nNgqgOBXD7+79kvqz2wfN7DlJJ9a7MQD1dUyv+c1soaSzJB39zOi1ZrbbzLaY2bwy26wzs14z6x3W\noZqaBZCfSYffzI6T9ANJ17n7AUm3SjpZ0lKNPTP45kTbuXuXu5fcvdSqthxaBpCHSYXfzFo1Fvzb\n3f1eSXL3AXcfcfdRSbdJWla/NgHkrWL4zcwkfUfSc+5+07jlneNWu1xSerpWAE1lMu/2nyfpC5Ke\nMrNd2bINktaY2VKNDf/1SfpyXToEUBeTebf/p5ImGjdMjukDaG58wg8IivADQRF+ICjCDwRF+IGg\nCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxUt357ozs9cl/XzcovmS9jesgWPTrL01\na18SvVUrz95OcvePTGbFhob/fTs363X3UmENJDRrb83al0Rv1SqqN572A0ERfiCoosPfVfD+U5q1\nt2btS6K3ahXSW6Gv+QEUp+gzP4CCFBJ+M1tpZs+b2UtmdkMRPZRjZn1m9pSZ7TKz3oJ72WJmg2b2\n9Lhl7Wb2oJm9mP2ecJq0gnrbaGb7smO3y8wuKai3BWb2YzN71syeMbO/yJYXeuwSfRVy3Br+tN/M\nWiS9IOliSXslPS5pjbs/29BGyjCzPkkldy98TNjMzpf0tqTt7r4kW/avkobcfVP2h3Oeu1/fJL1t\nlPR20TM3ZxPKdI6fWVrSZZK+qAKPXaKv1SrguBVx5l8m6SV33+PuhyXdJWlVAX00PXd/RNLQexav\nkrQtu71NY/95Gq5Mb03B3fvdfWd2+6CkozNLF3rsEn0VoojwnyjpF+Pu71VzTfntkh4ysyfMbF3R\nzUygI5s2XZJek9RRZDMTqDhzcyO9Z2bppjl21cx4nTfe8Hu/5e6+VNKnJV2dPb1tSj72mq2Zhmsm\nNXNzo0wws/RvFHnsqp3xOm9FhH+fpAXj7n8sW9YU3H1f9ntQ0n1qvtmHB45Okpr9Hiy4n99oppmb\nJ5pZWk1w7Jppxusiwv+4pMVmtsjMZkr6nKQdBfTxPmY2J3sjRmY2R9IKNd/swzskrc1ur5V0f4G9\n/JZmmbm53MzSKvjYNd2M1+7e8B9Jl2jsHf+XJf1tET2U6etkSU9mP88U3ZukOzX2NHBYY++NXCXp\nw5K6Jb0o6SFJ7U3U23clPSVpt8aC1llQb8s19pR+t6Rd2c8lRR+7RF+FHDc+4QcExRt+QFCEHwiK\n8ANBEX4gKMIPBEX4gaAIPxAU4QeC+n8DZI6NXofNrQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbb44629fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.imshow(X_train[0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The pretty keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "s = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "import keras.layers as ll\n",
    "\n",
    "model = Sequential(name=\"mlp\")\n",
    "\n",
    "model.add(ll.InputLayer([28, 28]))\n",
    "\n",
    "model.add(ll.Flatten())\n",
    "\n",
    "# network body\n",
    "model.add(ll.Dense(128))\n",
    "model.add(ll.Activation('relu'))\n",
    "\n",
    "model.add(ll.Dense(128))\n",
    "model.add(ll.Activation('sigmoid'))\n",
    "\n",
    "# output layer: 10 neurons for each class with softmax\n",
    "model.add(ll.Dense(10, activation='softmax'))\n",
    "\n",
    "# categorical_crossentropy is your good old crossentropy\n",
    "# but applied for one-hot-encoded vectors\n",
    "model.compile(\"adam\", \"categorical_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 28, 28)            0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 118,282\n",
      "Trainable params: 118,282\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model interface\n",
    "\n",
    "Keras models follow __Scikit-learn__'s interface of fit/predict with some notable extensions. Let's take a tour."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "50000/50000 [==============================] - 10s - loss: 0.3208 - acc: 0.9097 - val_loss: 0.1498 - val_acc: 0.9565\n",
      "Epoch 2/5\n",
      "50000/50000 [==============================] - 9s - loss: 0.1239 - acc: 0.9627 - val_loss: 0.1146 - val_acc: 0.9654\n",
      "Epoch 3/5\n",
      "50000/50000 [==============================] - 9s - loss: 0.0822 - acc: 0.9749 - val_loss: 0.0908 - val_acc: 0.9715\n",
      "Epoch 4/5\n",
      "50000/50000 [==============================] - 9s - loss: 0.0583 - acc: 0.9826 - val_loss: 0.1023 - val_acc: 0.9701\n",
      "Epoch 5/5\n",
      "50000/50000 [==============================] - 9s - loss: 0.0443 - acc: 0.9860 - val_loss: 0.0809 - val_acc: 0.9743\n"
     ]
    }
   ],
   "source": [
    "# fit(X,y) ships with a neat automatic logging.\n",
    "#          Highly customizable under the hood.\n",
    "model.fit(X_train, y_train,\n",
    "          validation_data=(X_val, y_val), epochs=5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2/2 [==============================] - 0s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[  3.12097157e-08,   1.20621407e-04,   6.54858522e-05,\n",
       "          9.99387026e-01,   4.85578596e-07,   3.77103715e-04,\n",
       "          4.28228866e-08,   6.14344174e-07,   4.76866517e-05,\n",
       "          9.76497745e-07],\n",
       "       [  3.94157581e-07,   4.19822328e-07,   2.16960825e-06,\n",
       "          2.47152475e-05,   1.85297466e-07,   4.35007860e-06,\n",
       "          1.17679997e-06,   6.05361308e-08,   9.99963999e-01,\n",
       "          2.46711511e-06]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# estimate probabilities P(y|x)\n",
    "model.predict_proba(X_val[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Save trained weights\n",
    "model.save(\"weights.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9408/10000 [===========================>..] - ETA: 0s\n",
      "Loss, Accuracy =  [0.074212815834768117, 0.97729999999999995]\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nLoss, Accuracy = \", model.evaluate(X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Whoops!\n",
    "So far our model is staggeringly inefficient. There is something wring with it. Guess, what?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9344/10000 [===========================>..] - ETA: 0s\n",
      "Test accuracy: 97.72999999999999 %\n",
      "Great job!\n"
     ]
    }
   ],
   "source": [
    "# Test score...\n",
    "test_predictions = model.predict_proba(X_test).argmax(axis=-1)\n",
    "test_answers = y_test.argmax(axis=-1)\n",
    "\n",
    "test_accuracy = np.mean(test_predictions==test_answers)\n",
    "\n",
    "print(\"\\nTest accuracy: {} %\".format(test_accuracy*100))\n",
    "\n",
    "assert test_accuracy>=0.92,\"Logistic regression can do better!\"\n",
    "assert test_accuracy>=0.975,\"Your network can do better!\"\n",
    "print(\"Great job!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "answer_submitter = grading.Grader(\"0ybD9ZxxEeea8A6GzH-6CA\")\n",
    "answer_submitter.set_answer(\"N56DR\", test_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-21-0639a365657e>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-21-0639a365657e>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    answer_submitter.submit(<your-email>, <your-assignment-token>)\u001b[0m\n\u001b[0m                            ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "answer_submitter.submit('siddharthdas8@gmail.com', <your-assignment-token>)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras + tensorboard\n",
    "\n",
    "Remember the interactive graphs from Tensorboard one notebook ago? \n",
    "\n",
    "Thing is, Keras can use tensorboard to show you a lot of useful information about the learning progress. Just take a look!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "! rm -r /tmp/tboard/**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard\n",
    "model.fit(X_train, y_train, validation_data=(X_val, y_val), \n",
    "          epochs=10,\n",
    "          callbacks=[TensorBoard(\"/tmp/tboard\")])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tips & tricks\n",
    "\n",
    "Here are some tips on what you could do. Don't worry, to reach the passing threshold you don't need to try all the ideas listed here, feel free to stop once you reach the 0.975 accuracy mark.\n",
    "\n",
    " * __Network size__\n",
    "   * More neurons, \n",
    "   * More layers, ([docs](https://keras.io/))\n",
    "\n",
    "   * Nonlinearities in the hidden layers\n",
    "     * tanh, relu, leaky relu, etc\n",
    "   * Larger networks may take more epochs to train, so don't discard your net just because it could didn't beat the baseline in 5 epochs.\n",
    "\n",
    "\n",
    " * __Early Stopping__\n",
    "   * Training for 100 epochs regardless of anything is probably a bad idea.\n",
    "   * Some networks converge over 5 epochs, others - over 500.\n",
    "   * Way to go: stop when validation score is 10 iterations past maximum\n",
    "     \n",
    "\n",
    " * __Faster optimization__\n",
    "   * rmsprop, nesterov_momentum, adam, adagrad and so on.\n",
    "     * Converge faster and sometimes reach better optima\n",
    "     * It might make sense to tweak learning rate/momentum, other learning parameters, batch size and number of epochs\n",
    "\n",
    "\n",
    " * __Regularize__ to prevent overfitting\n",
    "   * Add some L2 weight norm to the loss function, theano will do the rest\n",
    "     * Can be done manually or via - https://keras.io/regularizers/\n",
    "   \n",
    "   \n",
    " * __Data augmemntation__ - getting 5x as large dataset for free is a great deal\n",
    "   * https://keras.io/preprocessing/image/\n",
    "   * Zoom-in+slice = move\n",
    "   * Rotate+zoom(to remove black stripes)\n",
    "   * any other perturbations\n",
    "   * Simple way to do that (if you have PIL/Image): \n",
    "     * ```from scipy.misc import imrotate,imresize```\n",
    "     * and a few slicing\n",
    "   * Stay realistic. There's usually no point in flipping dogs upside down as that is not the way you usually see them."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "120px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}