3-layer-nn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "8bec90e6-23f2-4aae-8b4a-e9ef999391b7",
    "_uuid": "ae07d14dcb75d8188c49d57cdc54fc36f3397d02"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Alcohol</th>\n",
       "      <th>Malic acid</th>\n",
       "      <th>Ash</th>\n",
       "      <th>Alcalinity of ash</th>\n",
       "      <th>Magnesium</th>\n",
       "      <th>Total phenols</th>\n",
       "      <th>Flavanoids</th>\n",
       "      <th>Nonflavanoid phenols</th>\n",
       "      <th>Proanthocyanins</th>\n",
       "      <th>Color intensity</th>\n",
       "      <th>Hue</th>\n",
       "      <th>OD280/OD315 of diluted wines</th>\n",
       "      <th>Proline</th>\n",
       "      <th>Cultivar 1</th>\n",
       "      <th>Cultivar 2</th>\n",
       "      <th>Cultivar 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.518613</td>\n",
       "      <td>-0.562250</td>\n",
       "      <td>0.232053</td>\n",
       "      <td>-1.169593</td>\n",
       "      <td>1.913905</td>\n",
       "      <td>0.808997</td>\n",
       "      <td>1.034819</td>\n",
       "      <td>-0.659563</td>\n",
       "      <td>1.224884</td>\n",
       "      <td>0.251717</td>\n",
       "      <td>0.362177</td>\n",
       "      <td>1.847920</td>\n",
       "      <td>1.013009</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.246290</td>\n",
       "      <td>-0.499413</td>\n",
       "      <td>-0.827996</td>\n",
       "      <td>-2.490847</td>\n",
       "      <td>0.018145</td>\n",
       "      <td>0.568648</td>\n",
       "      <td>0.733629</td>\n",
       "      <td>-0.820719</td>\n",
       "      <td>-0.544721</td>\n",
       "      <td>-0.293321</td>\n",
       "      <td>0.406051</td>\n",
       "      <td>1.113449</td>\n",
       "      <td>0.965242</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.196879</td>\n",
       "      <td>0.021231</td>\n",
       "      <td>1.109334</td>\n",
       "      <td>-0.268738</td>\n",
       "      <td>0.088358</td>\n",
       "      <td>0.808997</td>\n",
       "      <td>1.215533</td>\n",
       "      <td>-0.498407</td>\n",
       "      <td>2.135968</td>\n",
       "      <td>0.269020</td>\n",
       "      <td>0.318304</td>\n",
       "      <td>0.788587</td>\n",
       "      <td>1.395148</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.691550</td>\n",
       "      <td>-0.346811</td>\n",
       "      <td>0.487926</td>\n",
       "      <td>-0.809251</td>\n",
       "      <td>0.930918</td>\n",
       "      <td>2.491446</td>\n",
       "      <td>1.466525</td>\n",
       "      <td>-0.981875</td>\n",
       "      <td>1.032155</td>\n",
       "      <td>1.186068</td>\n",
       "      <td>-0.427544</td>\n",
       "      <td>1.184071</td>\n",
       "      <td>2.334574</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.295700</td>\n",
       "      <td>0.227694</td>\n",
       "      <td>1.840403</td>\n",
       "      <td>0.451946</td>\n",
       "      <td>1.281985</td>\n",
       "      <td>0.808997</td>\n",
       "      <td>0.663351</td>\n",
       "      <td>0.226796</td>\n",
       "      <td>0.401404</td>\n",
       "      <td>-0.319276</td>\n",
       "      <td>0.362177</td>\n",
       "      <td>0.449601</td>\n",
       "      <td>-0.037874</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Alcohol  Malic acid       Ash  Alcalinity of ash  Magnesium  \\\n",
       "0  1.518613   -0.562250  0.232053          -1.169593   1.913905   \n",
       "1  0.246290   -0.499413 -0.827996          -2.490847   0.018145   \n",
       "2  0.196879    0.021231  1.109334          -0.268738   0.088358   \n",
       "3  1.691550   -0.346811  0.487926          -0.809251   0.930918   \n",
       "4  0.295700    0.227694  1.840403           0.451946   1.281985   \n",
       "\n",
       "   Total phenols  Flavanoids  Nonflavanoid phenols  Proanthocyanins  \\\n",
       "0       0.808997    1.034819             -0.659563         1.224884   \n",
       "1       0.568648    0.733629             -0.820719        -0.544721   \n",
       "2       0.808997    1.215533             -0.498407         2.135968   \n",
       "3       2.491446    1.466525             -0.981875         1.032155   \n",
       "4       0.808997    0.663351              0.226796         0.401404   \n",
       "\n",
       "   Color intensity       Hue  OD280/OD315 of diluted wines   Proline  \\\n",
       "0         0.251717  0.362177                      1.847920  1.013009   \n",
       "1        -0.293321  0.406051                      1.113449  0.965242   \n",
       "2         0.269020  0.318304                      0.788587  1.395148   \n",
       "3         1.186068 -0.427544                      1.184071  2.334574   \n",
       "4        -0.319276  0.362177                      0.449601 -0.037874   \n",
       "\n",
       "   Cultivar 1  Cultivar 2  Cultivar 3  \n",
       "0           1           0           0  \n",
       "1           1           0           0  \n",
       "2           1           0           0  \n",
       "3           1           0           0  \n",
       "4           1           0           0  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv('../input/W1data.csv')\n",
    "df.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "5212493d-23b7-40d4-b1d6-6ce70a3631ca",
    "_uuid": "17597c0a6ba90e709149929aeca1999e2e74df32",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get the wine labels\n",
    "y = df[['Cultivar 1', 'Cultivar 2', 'Cultivar 3']].values\n",
    "\n",
    "# Get inputs; we define our x and y here.\n",
    "X = df.drop(['Cultivar 1', 'Cultivar 2', 'Cultivar 3'], axis = 1)\n",
    "X.shape, y.shape # Print shapes just to check\n",
    "X = X.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "6b48153f-dd53-4bff-b04f-e6dfb036a59b",
    "_uuid": "6a2e1f0befbdb14f9085f58718ab64d9887e08a2",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#First we are importing all the libraries\n",
    "\n",
    "# Package imports\n",
    "# Matplotlib is a matlab like plotting library\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "# SciKitLearn is a useful machine learning utilities library\n",
    "import sklearn\n",
    "# The sklearn dataset module helps generating datasets\n",
    "import sklearn.datasets\n",
    "import sklearn.linear_model\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "90023c7a-8059-4117-98b3-82ceb3e5877a",
    "_uuid": "de660f4b64992b03558fc14ee176479b4b7afc29"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss after iteration 0 : 1.4504813382\n",
      "Accuracy after iteration 0 : 25.2808988764 %\n",
      "Loss after iteration 100 : 0.525521027288\n",
      "Accuracy after iteration 100 : 80.8988764045 %\n",
      "Loss after iteration 200 : 0.396422205604\n",
      "Accuracy after iteration 200 : 86.5168539326 %\n",
      "Loss after iteration 300 : 0.346940596919\n",
      "Accuracy after iteration 300 : 87.6404494382 %\n",
      "Loss after iteration 400 : 0.32062505328\n",
      "Accuracy after iteration 400 : 89.3258426966 %\n",
      "Loss after iteration 500 : 0.300576609266\n",
      "Accuracy after iteration 500 : 89.3258426966 %\n",
      "Loss after iteration 600 : 0.277575423902\n",
      "Accuracy after iteration 600 : 89.8876404494 %\n",
      "Loss after iteration 700 : 0.258034636799\n",
      "Accuracy after iteration 700 : 90.4494382022 %\n",
      "Loss after iteration 800 : 0.245189609147\n",
      "Accuracy after iteration 800 : 90.4494382022 %\n",
      "Loss after iteration 900 : 0.23453472997\n",
      "Accuracy after iteration 900 : 90.4494382022 %\n",
      "Loss after iteration 1000 : 0.224901853757\n",
      "Accuracy after iteration 1000 : 90.4494382022 %\n",
      "Loss after iteration 1100 : 0.216491159938\n",
      "Accuracy after iteration 1100 : 92.1348314607 %\n",
      "Loss after iteration 1200 : 0.208973335608\n",
      "Accuracy after iteration 1200 : 93.2584269663 %\n",
      "Loss after iteration 1300 : 0.201863997011\n",
      "Accuracy after iteration 1300 : 93.2584269663 %\n",
      "Loss after iteration 1400 : 0.180225501792\n",
      "Accuracy after iteration 1400 : 93.2584269663 %\n",
      "Loss after iteration 1500 : 0.166901822866\n",
      "Accuracy after iteration 1500 : 93.8202247191 %\n",
      "Loss after iteration 1600 : 0.156813419019\n",
      "Accuracy after iteration 1600 : 94.3820224719 %\n",
      "Loss after iteration 1700 : 0.148914311571\n",
      "Accuracy after iteration 1700 : 94.9438202247 %\n",
      "Loss after iteration 1800 : 0.142406536657\n",
      "Accuracy after iteration 1800 : 94.9438202247 %\n",
      "Loss after iteration 1900 : 0.135784561737\n",
      "Accuracy after iteration 1900 : 95.5056179775 %\n",
      "Loss after iteration 2000 : 0.131914667233\n",
      "Accuracy after iteration 2000 : 95.5056179775 %\n",
      "Loss after iteration 2100 : 0.129007075755\n",
      "Accuracy after iteration 2100 : 95.5056179775 %\n",
      "Loss after iteration 2200 : 0.126613715963\n",
      "Accuracy after iteration 2200 : 95.5056179775 %\n",
      "Loss after iteration 2300 : 0.12454049447\n",
      "Accuracy after iteration 2300 : 95.5056179775 %\n",
      "Loss after iteration 2400 : 0.122681638377\n",
      "Accuracy after iteration 2400 : 95.5056179775 %\n",
      "Loss after iteration 2500 : 0.120963378802\n",
      "Accuracy after iteration 2500 : 95.5056179775 %\n",
      "Loss after iteration 2600 : 0.119316076044\n",
      "Accuracy after iteration 2600 : 95.5056179775 %\n",
      "Loss after iteration 2700 : 0.117649064345\n",
      "Accuracy after iteration 2700 : 96.0674157303 %\n",
      "Loss after iteration 2800 : 0.115786808903\n",
      "Accuracy after iteration 2800 : 96.0674157303 %\n",
      "Loss after iteration 2900 : 0.113129107925\n",
      "Accuracy after iteration 2900 : 96.0674157303 %\n",
      "Loss after iteration 3000 : 0.10512216027\n",
      "Accuracy after iteration 3000 : 96.0674157303 %\n",
      "Loss after iteration 3100 : 0.0852832501938\n",
      "Accuracy after iteration 3100 : 97.191011236 %\n",
      "Loss after iteration 3200 : 0.073109370026\n",
      "Accuracy after iteration 3200 : 98.3146067416 %\n",
      "Loss after iteration 3300 : 0.0709484940955\n",
      "Accuracy after iteration 3300 : 98.3146067416 %\n",
      "Loss after iteration 3400 : 0.0664704165188\n",
      "Accuracy after iteration 3400 : 98.3146067416 %\n",
      "Loss after iteration 3500 : 0.0634969410056\n",
      "Accuracy after iteration 3500 : 98.3146067416 %\n",
      "Loss after iteration 3600 : 0.0620909181335\n",
      "Accuracy after iteration 3600 : 98.8764044944 %\n",
      "Loss after iteration 3700 : 0.0609872330329\n",
      "Accuracy after iteration 3700 : 98.8764044944 %\n",
      "Loss after iteration 3800 : 0.0598961421314\n",
      "Accuracy after iteration 3800 : 98.8764044944 %\n",
      "Loss after iteration 3900 : 0.0586844956054\n",
      "Accuracy after iteration 3900 : 98.8764044944 %\n",
      "Loss after iteration 4000 : 0.0572707668626\n",
      "Accuracy after iteration 4000 : 98.8764044944 %\n",
      "Loss after iteration 4100 : 0.0554444294155\n",
      "Accuracy after iteration 4100 : 98.8764044944 %\n",
      "Loss after iteration 4200 : 0.0518824879458\n",
      "Accuracy after iteration 4200 : 98.8764044944 %\n",
      "Loss after iteration 4300 : 0.043725816711\n",
      "Accuracy after iteration 4300 : 99.4382022472 %\n",
      "Loss after iteration 4400 : 0.0385472561531\n",
      "Accuracy after iteration 4400 : 99.4382022472 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f8a7f756518>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGHBJREFUeJzt3Xt0lPd95/H3VzdLIgYJLEAGyTIJ\ncfBVxIrrS+p4jZPajhtIaif2aU+ULKdkz8l23e0ttD3ZpDm92Od0N972tD2hsV3ak/oSxzHepMku\noXbdZWsc2eAbJAVjjLhKxojLDMwwM9/9Yx4JIY0k0DNoNL/5vM7RmXkePaP5+jF89OU3v+f5mbsj\nIiLhqip1ASIicn4p6EVEAqegFxEJnIJeRCRwCnoRkcAp6EVEAqegFxEJnIJeRCRwCnoRkcDVlLoA\ngIsuusg7OjpKXYaISFl5+eWX33X3lomOmxZB39HRQU9PT6nLEBEpK2b2ztkcp6EbEZHAKehFRAKn\noBcRCdyEQW9mj5hZn5m9MWzfbDNbb2bbo8fmaL+Z2V+Y2Q4ze83MPnw+ixcRkYmdTUf/d8DtI/at\nBja4+2JgQ7QNcAewOPpaBfxNccoUEZHJmjDo3f0F4L0Ru5cDa6Pna4EVw/b/vee9CDSZWWuxihUR\nkXM32TH6ee6+HyB6nBvtXwD0DjtuT7RPRERKpNjz6K3AvoJrFZrZKvLDO7S3txe5DBGR0jqeyvBa\n7wCv7T1CMpUZ87hlS+ZxTVvTea1lskF/0Mxa3X1/NDTTF+3fA7QNO24hsK/QD3D3NcAagK6uLi1c\nKyIF5XLTPx5y7uzoP86W3QNs3j3Alt4B/r3vGINLcluhFjgyd2b9tA36Z4Fu4IHocd2w/f/ZzB4H\nfgE4MjjEIyIykXQmx88OHGVL7+nAfPvdRKnLOidNjbV0tjVx51WtdLY30bmwiVmNtSWtacKgN7PH\ngFuAi8xsD/A18gH/pJmtBHYD90SH/xNwJ7ADSAJfPA81i0iJHDhyknVb9nJ8nKGIyTieyvDaniO8\nvvcI6UwOgJYLL6CzrYm7rm6lpmr6X/LTPqeBzrZmOuY0YuO18CUwYdC7+31jfGtZgWMd+HLcokRk\nenll92Ee3biLH72+n0zOqSpyjtXVVHHFxbP4/PWX0NnexNL2Zi6eVT/tArNcTYubmonI9JPO5PjR\nG/t5ZOMuXu0d4ML6Gr5wYwefv6GD9jmNpS5PzoGCXkTOcOh4in/ctJt/ePEd+o6lWHTRDL6x/Ap+\n5cMLmXGBIqMc6f+aiADw5r4jPLpxF8++uo90JsfNH2zhwbs7+NjiFqqKPVYjU0pBL1LBMtkcP9l2\nkEc27uKlt9+jobaaz3W10X1jBx+Y+75SlydFoqAXmcYGpxtu3j1A37GTRf/Z//T6AfYOnGBhcwN/\neOcSPvuRNmY1lHYqoBSfgl6Ckc052TK4uGY8fcdOnjGHfPh0w+oqK3jpeRzXXtLMV++6nI9fPo9q\nDc8ES0EvZcnd6X3vBJt7Dw+F4tZ9R0lnc6UurSguqKniqgWabijFoaCXaSOZzl8082rvAAMnThU8\nJpdzdvQdZ0vvAIcSaQDqa6u4ekET3TdeQlNj3VSWXHQz62vobGvmQ60XUls9/S8SkvKgoJeSyOWc\nt/qPs3nYMMXPDxxlcOSlbpyQa5vdwC2XzWVpexOdbU1cNl+hKDIeBb1MiUPHU2eMPb/aO8Cx6DL6\nmfU1XNPWxMdvXczStiauaWti9ozy7sxFphMFvRRdKpPlzX1H2RKF+ubew/S+dwLIf6D4ofkX8qnO\ni+lsy489L7pohuZpi5xHCnoZcvJUdlKvO3j09EyRzb0DbBv2oej8mfUsbW/i137hEpa2N3PVglk0\n1FUXs2wRmYCCvsJlsjnWbz3Ioxt38dKukStGnpuG2mquWjiLL97UEY2fNzN/Vn2RKhWRyVLQV6iB\nZJrHf9rLP/zbO0MXzPzGrR+gse7c/0jMaqjlmrZZXDbvQmr0oajItKOgrzDbDx7j0f+3i6df2cPJ\nUzmuXzSb//bLl3PbEl0wIxIqBX2FOHLiFH/6w2080dNLXU0Vn+5cwBdu6mBJ68xSlyYi55mCvgL8\n7zcP8NVn3uBQIs2Xbl7Elz72fk1fFKkgCvqA9R9L8fVn3+SHr+/n8taZPPKFj3DlglmlLktEplis\noDez+4FfBwz4W3d/yMxmA08AHcAu4LPufjhmnXIO3J2nX9nLN36wlRPpLL/7S5ex6uZFunpUpEJN\nOujN7EryIX8dkAZ+bGY/jPZtcPcHzGw1sBr4SjGKDUn/sdTQ3QlPpIu70PLW/UfZuOMQ117SzIO/\ncrXuKy5S4eJ09EuAF909CWBm/wJ8GlgO3BIdsxZ4ngoP+pOnoitFe6MrRXcfZs/h/JWiVQb1tcW9\ngKixrpqv//LlfP6GDl1xKiKxgv4N4E/MbA5wArgT6AHmuft+AHffb2ZzC73YzFYBqwDa29tjlDG9\nuDvvHEoOBfqW3gG27j/KqWz+bl0Xz6pnaXsz3Td00NnexJUX60pRETm/Jh307r7NzB4E1gPHgVeB\nsx6DcPc1wBqArq6uab9aRC7nPLNlL1v3HS38fYe3383fPvdwMn+L3ca6aq5aMIv/+NFLWdrWzNL2\nJubN1JWiIjK1Yn0Y6+4PAw8DmNmfAnuAg2bWGnXzrUBf/DJLa9e7CVY//Rov7syvqTnWaEhrUwO3\nLZnH0vZmOtua+OC89+lKUREpubizbua6e5+ZtQOfAW4ALgW6gQeix3WxqyyRbM555P++zX9f/3Nq\nq6p44DNX8bmPtGmVHxEpK3Hn0X8vGqM/BXzZ3Q+b2QPAk2a2EtgN3BO3yFL42YGjfOWp13h1zxFu\nWzKPP15xpW7QJSJlKe7QzS8W2HcIWBbn55ZSKpPlr557i79+bgezGmr5y/uWctfVreriRaRs6crY\nyMi7OX566QK+etflulWAiJS9ig/6kXdzvGHRHP7sM1dx8wdbSl2aiEhRVGTQ53LO8//ex6Mbd/Gv\n29/lgpoqVuhujiISqIoM+q//rzf5+397h/kz6/ndX7qM+65r1xCNiASr4oL+wJGTPPbSbu6+diF/\n9pmrdKMvEQlexaXcoxvfJptz7l+2WCEvIhWhopLu6MlTfGfTbj559cW0zW4sdTkiIlOiooL+Oy/u\n5ngqw5duXlTqUkREpkzFBH0qk+WRjW/z0Q9cpFWWRKSiVEzQP7N5L/3HUvynj72/1KWIiEypigj6\nXM751gs7ueLimdz0gTmlLkdEZEpVRND/ZNtBdvYn+NLH3q971ohIxamIoP/WCztpm93AnVfOL3Up\nIiJTLvig/+mu93j5ncP8+i8u0iIgIlKRgk++b/3LWzQ31nLPtW2lLkVEpCSCDvrtB4/xk219dN/Y\noQW4RaRiBR30a17YSX1tFZ+/oaPUpYiIlEysoDez/2pmb5rZG2b2mJnVm9mlZrbJzLab2RNmVpLb\nQh44cpJntuzl3o/ozpQiUtkmHfRmtgD4L0CXu18JVAP3Ag8C33T3xcBhYGUxCj1X3+3pJZNzVn70\n0lK8vYjItBF36KYGaDCzGqAR2A/cCjwVfX8tsCLme0xK37EUTQ21unmZiFS8SQe9u+8F/hzYTT7g\njwAvAwPunokO2wMsiFvkZCTSGRrrKu52+yIio8QZumkGlgOXAhcDM4A7ChzqY7x+lZn1mFlPf3//\nZMsYUzKVZcYFmmkjIhJn6OY24G1373f3U8DTwI1AUzSUA7AQ2Ffoxe6+xt273L2rpaX4C3GroxcR\nyYsT9LuB682s0fI3kFkGbAWeA+6OjukG1sUrcXKSaXX0IiIQb4x+E/kPXV8BXo9+1hrgK8BvmdkO\nYA7wcBHqPGeJlDp6ERGIuTi4u38N+NqI3TuB6+L83GJIprPM0NWwIiLhXhmbTGdovEAdvYhIsEGf\nSKmjFxGBQIM+l3NOnMpqjF5EhECD/sSpLIBm3YiIEGjQJ9L5C3PV0YuIBBr0yZQ6ehGRQUEGvTp6\nEZHTggz6ZDrq6BX0IiJhBn0ile/otXygiEigQT/U0WuMXkQkzKAf7Og1dCMiEmjQD3b0jRq6EREJ\nM+gHZ93M0L1uRETCDPpkKkuVwQU1Qf7niYickyCTMJHOMKOuhvx6KCIilS3IoE+msjRqxo2ICBBo\n0A929CIiEmjQJ9Pq6EVEBk066M3sMjPbMuzrqJn9ppnNNrP1ZrY9emwuZsFnQ+vFioicFmdx8J+7\ne6e7dwLXAkng+8BqYIO7LwY2RNtTSuvFioicVqyhm2XAW+7+DrAcWBvtXwusKNJ7nLWE1osVERlS\nrKC/F3gsej7P3fcDRI9zi/QeZ+2EOnoRkSGxg97M6oBPAd89x9etMrMeM+vp7++PW8YZNEYvInJa\nMTr6O4BX3P1gtH3QzFoBose+Qi9y9zXu3uXuXS0tLUUoY+jn5sfoNetGRAQoTtDfx+lhG4Bnge7o\neTewrgjvcdbS2RyZnKujFxGJxAp6M2sEPg48PWz3A8DHzWx79L0H4rzHuRpaL1Zj9CIiAMRqe909\nCcwZse8Q+Vk4JaH1YkVEzhTclbFD96LXGL2ICBBg0Gt1KRGRMwUX9FpdSkTkTMEF/VBHrytjRUSA\nAINeHb2IyJmCC3qtFysicqbggn5wHr06ehGRvOCCXvPoRUTOFFzQJ9NZ6murqK7SwuAiIhBg0CdS\nWi9WRGS44IJe68WKiJwpuKBXRy8icqbggv7Eqaxm3IiIDBNc0CdSGc2hFxEZJrigT6bV0YuIDBdc\n0CfSGqMXERkuuKBPpjTrRkRkuOCCPpHO6KpYEZFh4q4Z22RmT5nZz8xsm5ndYGazzWy9mW2PHpuL\nVexEsjnn5KmcxuhFRIaJ29H/T+DH7v4h4BpgG7Aa2ODui4EN0faUSKa1upSIyEiTDnozmwncDDwM\n4O5pdx8AlgNro8PWAiviFnm2tF6siMhocTr6RUA/8KiZbTazb5vZDGCeu+8HiB7nFnqxma0ysx4z\n6+nv749RxmlaL1ZEZLQ4QV8DfBj4G3dfCiQ4h2Ead1/j7l3u3tXS0hKjjNO0upSIyGhxgn4PsMfd\nN0XbT5EP/oNm1goQPfbFK/Hsab1YEZHRJh307n4A6DWzy6Jdy4CtwLNAd7SvG1gXq8JzoI5eRGS0\nuK3vbwDfMbM6YCfwRfK/PJ40s5XAbuCemO9x1rRerIjIaLES0d23AF0FvrUszs+dLK0XKyIyWlBX\nxiY0j15EZJSggl7z6EVERgsq6BOpDDVVRl11UP9ZIiKxBJWIg/eiN7NSlyIiMm0EFvRaXUpEZKSg\ngj6h1aVEREYJKuiTWi9WRGSUoIJeHb2IyGhBBX1Sq0uJiIwSVtCn1NGLiIwUVNAn0hldFSsiMkJQ\nQZ9MZXVVrIjICMEEvburoxcRKSCYoE9lcuRc97kRERkpmKDXerEiIoUFE/RaXUpEpLBggl6rS4mI\nFBYrFc1sF3AMyAIZd+8ys9nAE0AHsAv4rLsfjlfmxBJaXUpEpKBidPT/wd073X1wScHVwAZ3Xwxs\niLbPu6Q6ehGRgs7H0M1yYG30fC2w4jy8xyjq6EVECosb9A78HzN72cxWRfvmuft+gOhxbsz3OCtJ\nrRcrIlJQ3FS8yd33mdlcYL2Z/exsXxj9YlgF0N7eHrOM/J0rQfPoRURGitXRu/u+6LEP+D5wHXDQ\nzFoBose+MV67xt273L2rpaUlThkAnFBHLyJS0KSD3sxmmNmFg8+BTwBvAM8C3dFh3cC6uEWejcEx\n+oZadfQiIsPFaX/nAd+PFuKuAf7R3X9sZj8FnjSzlcBu4J74ZU4sfy/6aqqqtDC4iMhwkw56d98J\nXFNg/yFgWZyiJiO/upSGbURERgrmythkKqOplSIiBQQT9FovVkSksGCCPpnO6KpYEZECggn6hNaL\nFREpKJigT2p1KRGRgoIJ+oTWixURKSiYoFdHLyJSWDBBn0iroxcRKSSIoD+VzZHO5NTRi4gUEETQ\na71YEZGxBRL0Wl1KRGQsQQS9VpcSERlbEEGv1aVERMYWRNAPdfSadSMiMkoQQa+OXkRkbIEEfb6j\nn6GOXkRklECCPt/Ra+EREZHRggj6wTF6Dd2IiIwWO+jNrNrMNpvZD6LtS81sk5ltN7MnzKwufpnj\nG+zoGzS9UkRklGJ09PcD24ZtPwh8090XA4eBlUV4j3El0llqq426miD+gSIiUlSxktHMFgKfBL4d\nbRtwK/BUdMhaYEWc9zgb+fViNWwjIlJI3Bb4IeD3gFy0PQcYcPdMtL0HWBDzPSaUSGeZoWEbEZGC\nJh30ZnYX0OfuLw/fXeBQH+P1q8ysx8x6+vv7J1sGkB+jb9R9bkRECorT0d8EfMrMdgGPkx+yeQho\nMrPB1F0I7Cv0Yndf4+5d7t7V0tISo4z8rBt19CIihU066N399919obt3APcC/+zuvwo8B9wdHdYN\nrItd5QSSaY3Ri4iM5XxMU/kK8FtmtoP8mP3D5+E9zpBIZXVVrIjIGIrSBrv788Dz0fOdwHXF+Lln\nSx29iMjYgph4nkiroxcRGUsQQa959CIiYyv7oM/lnOQpzboRERlL2Qf9yUwWdzSPXkRkDGUf9Kfv\nXKmOXkSkkLIPet2LXkRkfAEEvVaXEhEZTwBBr45eRGQ8ZR/0Q2P06uhFRAoq+6AfWl2qVh29iEgh\nZR/06uhFRMZX9kGvMXoRkfGVfdAnNOtGRGRcZR/0yVQGM6ivUdCLiBRS9kGfSGdprK2mqqrQKoYi\nIlL2Qa/1YkVExlf2Qa/1YkVExlf2Qa/VpURExjfpoDezejN7ycxeNbM3zeyPov2XmtkmM9tuZk+Y\nWV3xyh1N68WKiIwvTkefAm5192uATuB2M7seeBD4prsvBg4DK+OXOTZ19CIi45t00Hve8WizNvpy\n4FbgqWj/WmBFrAonoPViRUTGF2uM3syqzWwL0AesB94CBtw9Ex2yB1gwxmtXmVmPmfX09/dPugat\nFysiMr5YQe/uWXfvBBYC1wFLCh02xmvXuHuXu3e1tLRMuoZEWrNuRETGU5RZN+4+ADwPXA80mdlg\ni70Q2FeM9xiL5tGLiIwvzqybFjNrip43ALcB24DngLujw7qBdXGLHEs6k+NU1tXRi4iMI04r3Aqs\nNbNq8r8wnnT3H5jZVuBxM/tjYDPwcBHqLOhEdEMzjdGLiIxt0gnp7q8BSwvs30l+vP68S0S3KNas\nGxGRsZX1lbFDq0upoxcRGVNZB/3Q6lIaoxcRGVN5B71WlxIRmVBZB31S68WKiEyorINeHb2IyMTK\nOuiTWi9WRGRCZR30iZQ6ehGRiZR10LfPbuSOK+fTqFk3IiJjKutW+BNXzOcTV8wvdRkiItNaWXf0\nIiIyMQW9iEjgFPQiIoFT0IuIBE5BLyISOAW9iEjgFPQiIoFT0IuIBM7cvdQ1YGb9wDuTfPlFwLtF\nLCcEOieF6byMpnMyWjmdk0vcvWWig6ZF0MdhZj3u3lXqOqYTnZPCdF5G0zkZLcRzoqEbEZHAKehF\nRAIXQtCvKXUB05DOSWE6L6PpnIwW3Dkp+zF6EREZXwgdvYiIjKOsg97Mbjezn5vZDjNbXep6SsHM\nHjGzPjN7Y9i+2Wa23sy2R4/NpaxxqplZm5k9Z2bbzOxNM7s/2l+x58XM6s3sJTN7NTonfxTtv9TM\nNkXn5Akzqyt1rVPNzKrNbLOZ/SDaDu6clG3Qm1k18FfAHcDlwH1mdnlpqyqJvwNuH7FvNbDB3RcD\nG6LtSpIBftvdlwDXA1+O/mxU8nlJAbe6+zVAJ3C7mV0PPAh8Mzonh4GVJayxVO4Htg3bDu6clG3Q\nA9cBO9x9p7ungceB5SWuacq5+wvAeyN2LwfWRs/XAiumtKgSc/f97v5K9PwY+b/EC6jg8+J5x6PN\n2ujLgVuBp6L9FXVOAMxsIfBJ4NvRthHgOSnnoF8A9A7b3hPtE5jn7vshH3rA3BLXUzJm1gEsBTZR\n4eclGqLYAvQB64G3gAF3z0SHVOLfoYeA3wNy0fYcAjwn5Rz0VmCfphDJEDN7H/A94Dfd/Wip6yk1\nd8+6eyewkPy/iJcUOmxqqyodM7sL6HP3l4fvLnBo2Z+Tcl4cfA/QNmx7IbCvRLVMNwfNrNXd95tZ\nK/kOrqKYWS35kP+Ouz8d7a748wLg7gNm9jz5zy+azKwm6mAr7e/QTcCnzOxOoB6YSb7DD+6clHNH\n/1NgcfQJeR1wL/BsiWuaLp4FuqPn3cC6EtYy5aJx1oeBbe7+P4Z9q2LPi5m1mFlT9LwBuI38ZxfP\nAXdHh1XUOXH333f3he7eQT4//tndf5UAz0lZXzAV/SZ+CKgGHnH3PylxSVPOzB4DbiF/x72DwNeA\nZ4AngXZgN3CPu4/8wDZYZvZR4F+B1zk99voH5MfpK/K8mNnV5D9YrCbf4D3p7t8ws0XkJzLMBjYD\nv+buqdJVWhpmdgvwO+5+V4jnpKyDXkREJlbOQzciInIWFPQiIoFT0IuIBE5BLyISOAW9iEjgFPQi\nIoFT0IuIBE5BLyISuP8Psikn9b/HuuwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8a7f7e6cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Now we define all our functions\n",
    "\n",
    "def softmax(z):\n",
    "    #Calculate exponent term first\n",
    "    exp_scores = np.exp(z)\n",
    "    return exp_scores / np.sum(exp_scores, axis=1, keepdims=True)\n",
    "\n",
    "def softmax_loss(y,y_hat):\n",
    "    # Clipping value\n",
    "    minval = 0.000000000001\n",
    "    # Number of samples\n",
    "    m = y.shape[0]\n",
    "    # Loss formula, note that np.sum sums up the entire matrix and therefore does the job of two sums from the formula\n",
    "    loss = -1/m * np.sum(y * np.log(y_hat.clip(min=minval)))\n",
    "    return loss\n",
    "\n",
    "def loss_derivative(y,y_hat):\n",
    "    return (y_hat-y)\n",
    "\n",
    "def tanh_derivative(x):\n",
    "    return (1 - np.power(x, 2))\n",
    "\n",
    "#FORWARD PROPAGATION\n",
    "# This is the forward propagation function\n",
    "def forward_prop(model,a0):\n",
    "\n",
    "    \n",
    "    # Load parameters from model\n",
    "    W1, b1, W2, b2, W3, b3 = model['W1'], model['b1'], model['W2'], model['b2'], model['W3'],model['b3']\n",
    "    \n",
    "    # Do the first Linear step \n",
    "    # Z1 is the input layer x times the dot product of the weights + our bias b\n",
    "    z1 = a0.dot(W1) + b1\n",
    "    \n",
    "    # Put it through the first activation function\n",
    "    a1 = np.tanh(z1)\n",
    "    \n",
    "    # Second linear step\n",
    "    z2 = a1.dot(W2) + b2\n",
    "    \n",
    "    # Second activation function\n",
    "    a2 = np.tanh(z2)\n",
    "    \n",
    "    #Third linear step\n",
    "    z3 = a2.dot(W3) + b3\n",
    "    \n",
    "    #For the Third linear activation function we use the softmax function, either the sigmoid of softmax should be used for the last layer\n",
    "    a3 = softmax(z3)\n",
    "    \n",
    "    #Store all results in these values\n",
    "    cache = {'a0':a0,'z1':z1,'a1':a1,'z2':z2,'a2':a2,'a3':a3,'z3':z3}\n",
    "    return cache\n",
    "\n",
    "# This is the backward propagation function\n",
    "def backward_prop(model,cache,y):\n",
    "\n",
    "    # Load parameters from model\n",
    "    W1, b1, W2, b2, W3, b3 = model['W1'], model['b1'], model['W2'], model['b2'],model['W3'],model['b3']\n",
    "    \n",
    "    # Load forward propagation results\n",
    "    a0,a1, a2,a3 = cache['a0'],cache['a1'],cache['a2'],cache['a3']\n",
    "    \n",
    "    # Get number of samples\n",
    "    m = y.shape[0]\n",
    "    \n",
    "    # Calculate loss derivative with respect to output\n",
    "    dz3 = loss_derivative(y=y,y_hat=a3)\n",
    "\n",
    "    # Calculate loss derivative with respect to second layer weights\n",
    "    dW3 = 1/m*(a2.T).dot(dz3) #dW2 = 1/m*(a1.T).dot(dz2) \n",
    "    \n",
    "    # Calculate loss derivative with respect to second layer bias\n",
    "    db3 = 1/m*np.sum(dz3, axis=0)\n",
    "    \n",
    "    # Calculate loss derivative with respect to first layer\n",
    "    dz2 = np.multiply(dz3.dot(W3.T) ,tanh_derivative(a2))\n",
    "    \n",
    "    # Calculate loss derivative with respect to first layer weights\n",
    "    dW2 = 1/m*np.dot(a1.T, dz2)\n",
    "    \n",
    "    # Calculate loss derivative with respect to first layer bias\n",
    "    db2 = 1/m*np.sum(dz2, axis=0)\n",
    "    \n",
    "    dz1 = np.multiply(dz2.dot(W2.T),tanh_derivative(a1))\n",
    "    \n",
    "    dW1 = 1/m*np.dot(a0.T,dz1)\n",
    "    \n",
    "    db1 = 1/m*np.sum(dz1,axis=0)\n",
    "    \n",
    "    # Store gradients\n",
    "    grads = {'dW3':dW3, 'db3':db3, 'dW2':dW2,'db2':db2,'dW1':dW1,'db1':db1}\n",
    "    return grads\n",
    "\n",
    "#TRAINING PHASE\n",
    "def initialize_parameters(nn_input_dim,nn_hdim,nn_output_dim):\n",
    "    # First layer weights\n",
    "    W1 = 2 *np.random.randn(nn_input_dim, nn_hdim) - 1\n",
    "    \n",
    "    # First layer bias\n",
    "    b1 = np.zeros((1, nn_hdim))\n",
    "    \n",
    "    # Second layer weights\n",
    "    W2 = 2 * np.random.randn(nn_hdim, nn_hdim) - 1\n",
    "    \n",
    "    # Second layer bias\n",
    "    b2 = np.zeros((1, nn_hdim))\n",
    "    W3 = 2 * np.random.rand(nn_hdim, nn_output_dim) - 1\n",
    "    b3 = np.zeros((1,nn_output_dim))\n",
    "    \n",
    "    \n",
    "    # Package and return model\n",
    "    model = { 'W1': W1, 'b1': b1, 'W2': W2, 'b2': b2,'W3':W3,'b3':b3}\n",
    "    return model\n",
    "\n",
    "def update_parameters(model,grads,learning_rate):\n",
    "    # Load parameters\n",
    "    W1, b1, W2, b2,b3,W3 = model['W1'], model['b1'], model['W2'], model['b2'],model['b3'],model[\"W3\"]\n",
    "    \n",
    "    # Update parameters\n",
    "    W1 -= learning_rate * grads['dW1']\n",
    "    b1 -= learning_rate * grads['db1']\n",
    "    W2 -= learning_rate * grads['dW2']\n",
    "    b2 -= learning_rate * grads['db2']\n",
    "    W3 -= learning_rate * grads['dW3']\n",
    "    b3 -= learning_rate * grads['db3']\n",
    "    \n",
    "    # Store and return parameters\n",
    "    model = { 'W1': W1, 'b1': b1, 'W2': W2, 'b2': b2, 'W3':W3,'b3':b3}\n",
    "    return model\n",
    "\n",
    "def predict(model, x):\n",
    "    # Do forward pass\n",
    "    c = forward_prop(model,x)\n",
    "    #get y_hat\n",
    "    y_hat = np.argmax(c['a3'], axis=1)\n",
    "    return y_hat\n",
    "\n",
    "def calc_accuracy(model,x,y):\n",
    "    # Get total number of examples\n",
    "    m = y.shape[0]\n",
    "    # Do a prediction with the model\n",
    "    pred = predict(model,x)\n",
    "    # Ensure prediction and truth vector y have the same shape\n",
    "    pred = pred.reshape(y.shape)\n",
    "    # Calculate the number of wrong examples\n",
    "    error = np.sum(np.abs(pred-y))\n",
    "    # Calculate accuracy\n",
    "    return (m - error)/m * 100\n",
    "losses = []\n",
    "\n",
    "def train(model,X_,y_,learning_rate, epochs=20000, print_loss=False):\n",
    "    # Gradient descent. Loop over epochs\n",
    "    for i in range(0, epochs):\n",
    "\n",
    "        # Forward propagation\n",
    "        cache = forward_prop(model,X_)\n",
    "        #a1, probs = cache['a1'],cache['a2']\n",
    "        # Backpropagation\n",
    "        \n",
    "        grads = backward_prop(model,cache,y_)\n",
    "        # Gradient descent parameter update\n",
    "        # Assign new parameters to the model\n",
    "        model = update_parameters(model=model,grads=grads,learning_rate=learning_rate)\n",
    "    \n",
    "        # Pring loss & accuracy every 100 iterations\n",
    "        if print_loss and i % 100 == 0:\n",
    "            a3 = cache['a3']\n",
    "            print('Loss after iteration',i,':',softmax_loss(y_,a3))\n",
    "            y_hat = predict(model,X_)\n",
    "            y_true = y_.argmax(axis=1)\n",
    "            print('Accuracy after iteration',i,':',accuracy_score(y_pred=y_hat,y_true=y_true)*100,'%')\n",
    "            losses.append(accuracy_score(y_pred=y_hat,y_true=y_true)*100)\n",
    "    return model\n",
    "np.random.seed(0)\n",
    "# This is what we return at the end\n",
    "model = initialize_parameters(nn_input_dim=13, nn_hdim= 5, nn_output_dim= 3)\n",
    "model = train(model,X,y,learning_rate=0.07,epochs=4500,print_loss=True)\n",
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cf85a92b-2bc6-4b57-8cd6-53b523add48a",
    "_uuid": "6aca14a72b394b8bc0d83fae7134fd4f8bcbfce2",
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}