convnet1d_CEPH.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline \n",
    "import tensorflow as tf\n",
    "\n",
    "from parser_CEPH import CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = tf.ConfigProto()\n",
    "#config.gpu_options.per_process_gpu_memory_fraction = 0.4\n",
    "tf.set_random_seed(1)\n",
    "sess = tf.InteractiveSession(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Load dataset\n",
    "batch_size = 64\n",
    "cifar10 = CIFAR10(batch_size=batch_size, validation_proportion=0.1, augment_data=False, file='data_CEPH.csv')\n",
    "\n",
    "SUMMARIES_DIR = './summaries/convnet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model blocks\n",
    "def conv_layer(input_tensor, kernel_shape, layer_name):\n",
    "    # input_tensor b01c\n",
    "    # kernel_shape 01-in-out\n",
    "    weights = tf.get_variable(\"weights\", kernel_shape,\n",
    "                               initializer = tf.contrib.layers.xavier_initializer_conv2d())\n",
    "    biases = tf.get_variable(\"biases\", [kernel_shape[3]],\n",
    "                             initializer=tf.constant_initializer(0.05))\n",
    "    \n",
    "    tf.summary.histogram(layer_name + \"/weights\", weights)\n",
    "    tf.summary.histogram(layer_name + \"/biases\", biases)\n",
    "    \n",
    "    # Other options are to use He et. al init. for weights and 0.01 \n",
    "    # to init. biases.\n",
    "    conv = tf.nn.conv2d(input_tensor, weights, \n",
    "                       strides = [1, 1, 1, 1], padding='SAME')\n",
    "    return tf.nn.relu(conv + biases)\n",
    "\n",
    "def fc_layer(input_tensor, weights_shape, layer_name):\n",
    "    # weights_shape in-out\n",
    "    weights = tf.get_variable(\"weights\", weights_shape,\n",
    "                              initializer = tf.contrib.layers.xavier_initializer())\n",
    "    biases = tf.get_variable(\"biases\", [weights_shape[1]],\n",
    "                             initializer=tf.constant_initializer(0.0))\n",
    "    tf.summary.histogram(layer_name + \"/weights\", weights)\n",
    "    tf.summary.histogram(layer_name + \"/biases\", biases)\n",
    "    mult_out = tf.matmul(input_tensor, weights)\n",
    "    return tf.nn.relu(mult_out+biases)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model\n",
    "model_input = tf.placeholder(tf.float32, name='model_input', \n",
    "                             shape=(batch_size,1,200,1))\n",
    "tf.summary.image('input', model_input, 10)\n",
    "\n",
    "keep_prob = tf.placeholder(tf.float32, name='dropout_prob', shape=())\n",
    "\n",
    "target = tf.placeholder(tf.float32, name='target', shape=(batch_size, 1))\n",
    "\n",
    "# First convolution layer\n",
    "layer_name = 'conv1'\n",
    "with tf.variable_scope(layer_name):\n",
    "    conv1_out = conv_layer(model_input, [1, 3, 1, 20], layer_name)\n",
    "# First pooling layer\n",
    "with tf.name_scope('pool1'):\n",
    "    pool1_out = tf.nn.avg_pool(conv1_out, ksize=[1, 2, 2, 1],\n",
    "                strides=[1, 2, 2, 1], padding='SAME',\n",
    "                name='pool1')\n",
    "    \n",
    "\n",
    "# Second convolution layer\n",
    "layer_name = 'conv2'\n",
    "with tf.variable_scope(layer_name):\n",
    "    conv2_out = conv_layer(pool1_out, [1, 3, 20, 20], layer_name)\n",
    "# Second pooling layer\n",
    "with tf.name_scope('pool2'):\n",
    "    pool2_out = tf.nn.avg_pool(conv2_out, ksize=[1, 2, 2, 1],\n",
    "                            strides=[1, 2, 2, 1], padding='SAME',\n",
    "                            name='pool2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,_,dim3,dim4 = pool2_out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pool2_out_flat = tf.reshape(pool2_out, [-1, 1*dim3.value*dim4.value], name='pool2_flat')\n",
    "# First fully connected layer\n",
    "layer_name = 'fc1'\n",
    "with tf.variable_scope(layer_name):\n",
    "    fc1_out = fc_layer(pool2_out_flat, [1*dim3.value*dim4.value, 500], layer_name)\n",
    "fc1_out_drop = tf.nn.dropout(fc1_out, keep_prob)\n",
    "\n",
    "# Second fully connected layer\n",
    "layer_name = 'fc2'\n",
    "with tf.variable_scope(layer_name):\n",
    "    fc2_out = fc_layer(fc1_out_drop, [500, 250], layer_name)\n",
    "fc2_out_drop = tf.nn.dropout(fc2_out, keep_prob)\n",
    "\n",
    "\n",
    "# Third fully connected layer\n",
    "layer_name = 'fc3'\n",
    "with tf.variable_scope(layer_name):\n",
    "    fc3_out = fc_layer(fc2_out_drop, [250,1], layer_name)\n",
    "model_output = fc3_out\n",
    "\n",
    "\n",
    "with tf.name_scope('loss_function'):\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "        tf.nn.softmax_cross_entropy_with_logits(logits=model_output, labels=target,\n",
    "                                           name='cross_entropy'))\n",
    "    tf.summary.scalar('cross_entropy', cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimization\n",
    "with tf.name_scope('optimizer'):\n",
    "    optimizer = tf.train.AdamOptimizer()\n",
    "    grads_vars = optimizer.compute_gradients(cross_entropy)\n",
    "    optimizer.apply_gradients(grads_vars)\n",
    "    train_step = optimizer.minimize(cross_entropy)\n",
    "\n",
    "# Metrics\n",
    "correct_prediction = tf.equal(tf.argmax(model_output, 1),\n",
    "                             tf.argmax(target, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy')\n",
    "tf.summary.scalar('accuracy', accuracy)\n",
    "\n",
    "# Useful training functions\n",
    "def validate():\n",
    "    batches = cifar10.getValidationSet(asBatches=True)\n",
    "    accs = []\n",
    "    for batch in batches:\n",
    "        data, labels = batch\n",
    "        acc = sess.run(accuracy,\n",
    "                       feed_dict={\n",
    "                model_input: data,\n",
    "                target: labels,\n",
    "                keep_prob: 1.0\n",
    "            })\n",
    "        accs.append(acc)\n",
    "    mean_acc = np.array(accs).mean()\n",
    "    summary = sess.run(merged,\n",
    "                       feed_dict={\n",
    "                model_input: data,\n",
    "                target: labels,\n",
    "                keep_prob: 1.0\n",
    "            })\n",
    "    return summary, mean_acc\n",
    "def test():\n",
    "    batches = cifar10.getTestSet(asBatches=True)\n",
    "    accs = []\n",
    "    for batch in batches:\n",
    "        data, labels = batch\n",
    "        acc = sess.run(accuracy,\n",
    "                       feed_dict={\n",
    "                model_input: data,\n",
    "                target: labels,\n",
    "                keep_prob: 1.0\n",
    "            })\n",
    "        accs.append(acc)\n",
    "    mean_acc = np.array(accs).mean()\n",
    "    return mean_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable variables\n",
      "conv1/weights:0\n",
      "conv1/biases:0\n",
      "conv2/weights:0\n",
      "conv2/biases:0\n",
      "fc1/weights:0\n",
      "fc1/biases:0\n",
      "fc2/weights:0\n",
      "fc2/biases:0\n",
      "fc3/weights:0\n",
      "fc3/biases:0\n",
      "Epoch 0, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.002403545379638672 minutes\n",
      "Epoch 1, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.008164493242899577 minutes\n",
      "Epoch 2, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.014445491631825765 minutes\n",
      "Epoch 3, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.02064401308695475 minutes\n",
      "Epoch 4, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.02663248380025228 minutes\n",
      "Epoch 5, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.032506863276163735 minutes\n",
      "Epoch 6, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.03855424722035726 minutes\n",
      "Epoch 7, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.0445386528968811 minutes\n",
      "Epoch 8, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.050802151362101235 minutes\n",
      "Epoch 9, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.05663800636927287 minutes\n",
      "Epoch 10, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.06256923278172812 minutes\n",
      "Epoch 11, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.06782223383585612 minutes\n",
      "Epoch 12, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.0741487979888916 minutes\n",
      "Epoch 13, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.08002892335255941 minutes\n",
      "Epoch 14, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.08603997230529785 minutes\n",
      "Epoch 15, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.09129311641057332 minutes\n",
      "Epoch 16, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.09713435967763265 minutes\n",
      "Epoch 17, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.10326544443766277 minutes\n",
      "Epoch 18, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.10969090461730957 minutes\n",
      "Epoch 19, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.11692750453948975 minutes\n",
      "Epoch 20, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.12397727966308594 minutes\n",
      "Epoch 21, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.13032718102137247 minutes\n",
      "Epoch 22, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.13688052892684938 minutes\n",
      "Epoch 23, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.14300076564153036 minutes\n",
      "Epoch 24, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.1507274587949117 minutes\n",
      "Epoch 25, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.1583932399749756 minutes\n",
      "Epoch 26, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.16634389162063598 minutes\n",
      "Epoch 27, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.17237751086552938 minutes\n",
      "Epoch 28, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.17810091972351075 minutes\n",
      "Epoch 29, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.18334471384684245 minutes\n",
      "Epoch 30, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.18905797402064006 minutes\n",
      "Epoch 31, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.19514979918797812 minutes\n",
      "Epoch 32, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.20151943763097127 minutes\n",
      "Epoch 33, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.20754302740097047 minutes\n",
      "Epoch 34, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.21351825793584187 minutes\n",
      "Epoch 35, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.2199676513671875 minutes\n",
      "Epoch 36, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.2257407267888387 minutes\n",
      "Epoch 37, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.23207250038782756 minutes\n",
      "Epoch 38, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.23775415817896525 minutes\n",
      "Epoch 39, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.2429200013478597 minutes\n",
      "Epoch 40, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.24966510136922201 minutes\n",
      "Epoch 41, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.25499242544174194 minutes\n",
      "Epoch 42, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.2606276551882426 minutes\n",
      "Epoch 43, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.2659954071044922 minutes\n",
      "Epoch 44, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.27164427439371747 minutes\n",
      "Epoch 45, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.276902437210083 minutes\n",
      "Epoch 46, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.2830056389172872 minutes\n",
      "Epoch 47, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.28847034374872843 minutes\n",
      "Epoch 48, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.2942366123199463 minutes\n",
      "Epoch 49, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.29964938163757326 minutes\n",
      "Epoch 50, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.30533660252889 minutes\n",
      "Epoch 51, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.310940690835317 minutes\n",
      "Epoch 52, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.31778544584910073 minutes\n",
      "Epoch 53, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.3228318691253662 minutes\n",
      "Epoch 54, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.3289015769958496 minutes\n",
      "Epoch 55, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.334071679910024 minutes\n",
      "Epoch 56, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.339929993947347 minutes\n",
      "Epoch 57, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.34542269706726075 minutes\n",
      "Epoch 58, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.352272633711497 minutes\n",
      "Epoch 59, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.35745317141215005 minutes\n",
      "Epoch 60, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.3632738629976908 minutes\n",
      "Epoch 61, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.3689036726951599 minutes\n",
      "Epoch 62, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.37520822683970134 minutes\n",
      "Epoch 63, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.38094934622446697 minutes\n",
      "Epoch 64, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.3876138647397359 minutes\n",
      "Epoch 65, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.3931745131810506 minutes\n",
      "Epoch 66, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.39956955512364706 minutes\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.40536961555480955 minutes\n",
      "Epoch 68, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.41331196228663125 minutes\n",
      "Epoch 69, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.4194904009501139 minutes\n",
      "Epoch 70, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.4251163442929586 minutes\n",
      "Epoch 71, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.43057217200597125 minutes\n",
      "Epoch 72, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.4364224553108215 minutes\n",
      "Epoch 73, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.44173504908879596 minutes\n",
      "Epoch 74, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.44847561915715534 minutes\n",
      "Epoch 75, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.45475480556488035 minutes\n",
      "Epoch 76, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.4602440595626831 minutes\n",
      "Epoch 77, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.46587979793548584 minutes\n",
      "Epoch 78, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.47275710900624596 minutes\n",
      "Epoch 79, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.478609029452006 minutes\n",
      "Epoch 80, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.48670939207077024 minutes\n",
      "Epoch 81, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.4922893802324931 minutes\n",
      "Epoch 82, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.4980483651161194 minutes\n",
      "Epoch 83, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5036761840184529 minutes\n",
      "Epoch 84, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5095167438189189 minutes\n",
      "Epoch 85, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5149584849675496 minutes\n",
      "Epoch 86, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5212269385655721 minutes\n",
      "Epoch 87, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5267287373542786 minutes\n",
      "Epoch 88, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5328088363011678 minutes\n",
      "Epoch 89, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5383403023084005 minutes\n",
      "Epoch 90, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5446946700414022 minutes\n",
      "Epoch 91, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5504981319109599 minutes\n",
      "Epoch 92, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5561485687891642 minutes\n",
      "Epoch 93, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5636455456415812 minutes\n",
      "Epoch 94, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5695618549982707 minutes\n",
      "Epoch 95, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5768821517626445 minutes\n",
      "Epoch 96, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5829334775606791 minutes\n",
      "Epoch 97, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5887067317962646 minutes\n",
      "Epoch 98, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.5956068436304728 minutes\n",
      "Epoch 99, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6009840965270996 minutes\n",
      "Epoch 100, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.608009147644043 minutes\n",
      "Epoch 101, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6140828688939413 minutes\n",
      "Epoch 102, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.619987674554189 minutes\n",
      "Epoch 103, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6256338278452556 minutes\n",
      "Epoch 104, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6318786263465881 minutes\n",
      "Epoch 105, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6376808444658916 minutes\n",
      "Epoch 106, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6441182216008504 minutes\n",
      "Epoch 107, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6499531110127766 minutes\n",
      "Epoch 108, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6559862971305848 minutes\n",
      "Epoch 109, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.662106204032898 minutes\n",
      "Epoch 110, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.668036965529124 minutes\n",
      "Epoch 111, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6736563603083293 minutes\n",
      "Epoch 112, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6795995831489563 minutes\n",
      "Epoch 113, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6854289929072063 minutes\n",
      "Epoch 114, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6917186379432678 minutes\n",
      "Epoch 115, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.6970701932907104 minutes\n",
      "Epoch 116, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7028519352277119 minutes\n",
      "Epoch 117, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7081724842389424 minutes\n",
      "Epoch 118, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7140404939651489 minutes\n",
      "Epoch 119, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7201195955276489 minutes\n",
      "Epoch 120, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7258187135060629 minutes\n",
      "Epoch 121, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7310497403144837 minutes\n",
      "Epoch 122, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7371117194493612 minutes\n",
      "Epoch 123, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.742353638013204 minutes\n",
      "Epoch 124, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7480634013811748 minutes\n",
      "Epoch 125, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7532800833384196 minutes\n",
      "Epoch 126, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7590537349383036 minutes\n",
      "Epoch 127, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7652257561683655 minutes\n",
      "Epoch 128, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7716076731681824 minutes\n",
      "Epoch 129, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7769639809926351 minutes\n",
      "Epoch 130, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.785531481107076 minutes\n",
      "Epoch 131, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7912879387537638 minutes\n",
      "Epoch 132, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.7971346060434977 minutes\n",
      "Epoch 133, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8026053667068481 minutes\n",
      "Epoch 134, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8085037748018901 minutes\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 135, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8163323005040487 minutes\n",
      "Epoch 136, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8226948022842407 minutes\n",
      "Epoch 137, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8279746373494467 minutes\n",
      "Epoch 138, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8334732254346212 minutes\n",
      "Epoch 139, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8388220787048339 minutes\n",
      "Epoch 140, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.844518772761027 minutes\n",
      "Epoch 141, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8503704388936361 minutes\n",
      "Epoch 142, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8567045331001282 minutes\n",
      "Epoch 143, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8642668485641479 minutes\n",
      "Epoch 144, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8703650633494059 minutes\n",
      "Epoch 145, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8776036699612936 minutes\n",
      "Epoch 146, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8842192610104879 minutes\n",
      "Epoch 147, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8903314193089803 minutes\n",
      "Epoch 148, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.8965908050537109 minutes\n",
      "Epoch 149, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9022596001625061 minutes\n",
      "Epoch 150, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9084604144096374 minutes\n",
      "Epoch 151, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.914230223496755 minutes\n",
      "Epoch 152, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9202324628829956 minutes\n",
      "Epoch 153, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9257718761761983 minutes\n",
      "Epoch 154, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9320298075675965 minutes\n",
      "Epoch 155, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9393747727076213 minutes\n",
      "Epoch 156, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9458657622337341 minutes\n",
      "Epoch 157, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9515922784805297 minutes\n",
      "Epoch 158, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9575782299041748 minutes\n",
      "Epoch 159, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9630536278088887 minutes\n",
      "Epoch 160, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9689606189727783 minutes\n",
      "Epoch 161, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9747403621673584 minutes\n",
      "Epoch 162, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9807392120361328 minutes\n",
      "Epoch 163, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9861440062522888 minutes\n",
      "Epoch 164, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9921601573626201 minutes\n",
      "Epoch 165, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 0.9974723219871521 minutes\n",
      "Epoch 166, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0035656889279683 minutes\n",
      "Epoch 167, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0093870043754578 minutes\n",
      "Epoch 168, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0155951142311097 minutes\n",
      "Epoch 169, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0213605761528015 minutes\n",
      "Epoch 170, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0272473533948263 minutes\n",
      "Epoch 171, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.032974636554718 minutes\n",
      "Epoch 172, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0389129996299744 minutes\n",
      "Epoch 173, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0440862933794657 minutes\n",
      "Epoch 174, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0498376091321309 minutes\n",
      "Epoch 175, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0549170970916748 minutes\n",
      "Epoch 176, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0604746778806051 minutes\n",
      "Epoch 177, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0679747740427652 minutes\n",
      "Epoch 178, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0735396226247151 minutes\n",
      "Epoch 179, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0812654773394266 minutes\n",
      "Epoch 180, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.0874151905377707 minutes\n",
      "Epoch 181, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.093333101272583 minutes\n",
      "Epoch 182, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1013048887252808 minutes\n",
      "Epoch 183, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1076850136121115 minutes\n",
      "Epoch 184, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1135392904281616 minutes\n",
      "Epoch 185, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1218602895736693 minutes\n",
      "Epoch 186, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1284253319104514 minutes\n",
      "Epoch 187, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1338977138201396 minutes\n",
      "Epoch 188, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1405261476834616 minutes\n",
      "Epoch 189, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1464113076527913 minutes\n",
      "Epoch 190, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1525214473406473 minutes\n",
      "Epoch 191, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1605631073315938 minutes\n",
      "Epoch 192, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1663170138994852 minutes\n",
      "Epoch 193, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.171746269861857 minutes\n",
      "Epoch 194, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1773518085479737 minutes\n",
      "Epoch 195, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1829651117324829 minutes\n",
      "Epoch 196, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1889216661453248 minutes\n",
      "Epoch 197, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.1948591351509095 minutes\n",
      "Epoch 198, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.2010347882906596 minutes\n",
      "Epoch 199, training loss 0.000000, accuracy 1.000000\n",
      "Validation accuracy 1.000000\n",
      "Time elapsed 1.2064796447753907 minutes\n",
      "Testing set accuracy 1.000000\n"
     ]
    }
   ],
   "source": [
    "# Train model\n",
    "merged = tf.summary.merge_all()\n",
    "train_writer = tf.summary.FileWriter(SUMMARIES_DIR+'/train',\n",
    "                                     sess.graph)\n",
    "validation_writer = tf.summary.FileWriter(SUMMARIES_DIR+'/validation')\n",
    "sess.run(tf.global_variables_initializer())\n",
    "cifar10.reset()\n",
    "print(\"Trainable variables\")\n",
    "for n in tf.trainable_variables():\n",
    "    print(n.name)\n",
    "\n",
    "epochs = 200\n",
    "mean_gradients = np.zeros([len(tf.trainable_variables()), epochs])\n",
    "std_gradients = np.zeros([len(tf.trainable_variables()), epochs])\n",
    "\n",
    "t_i = time.time()\n",
    "n_batches = cifar10.n_batches\n",
    "while cifar10.getEpoch() < epochs:\n",
    "    epoch = cifar10.getEpoch()\n",
    "    batch, batch_idx = cifar10.nextBatch()\n",
    "    batch_data = batch[0]\n",
    "    batch_labels = batch[1]\n",
    "    \n",
    "    # just a training iteration\n",
    "    _ = sess.run(train_step,\n",
    "                feed_dict={\n",
    "            model_input: batch_data,\n",
    "            target: batch_labels,\n",
    "            keep_prob: 0.5\n",
    "        })\n",
    "    \n",
    "    step = batch_idx+epoch*n_batches\n",
    "    \n",
    "    # Write training summary\n",
    "    if step%50==0:\n",
    "        summary = sess.run(merged,\n",
    "                          feed_dict={\n",
    "                model_input: batch_data,\n",
    "                target: batch_labels,\n",
    "                keep_prob: 0.5 # set to 1.0 at inference time\n",
    "            })\n",
    "        train_writer.add_summary(summary, step)\n",
    "        \n",
    "    # gradient (by layer) statistics over last training batch & validation summary\n",
    "    if batch_idx==0:\n",
    "        loss, acc, grads = sess.run((cross_entropy, accuracy, grads_vars), \n",
    "                      feed_dict={\n",
    "            model_input: batch_data,\n",
    "            target: batch_labels,\n",
    "            keep_prob: 1.0\n",
    "        })\n",
    "\n",
    "        for layer in range(len(tf.trainable_variables())):\n",
    "            mean_gradients[layer, epoch] = np.mean(np.abs(grads[layer][0]))\n",
    "            std_gradients[layer, epoch] = np.std(np.abs(grads[layer][0]))\n",
    "        print(\"Epoch %d, training loss %f, accuracy %f\" % (epoch, loss, acc))\n",
    "        \n",
    "        summary, validation_accuracy = validate()\n",
    "        validation_writer.add_summary(summary, step)\n",
    "        print(\"Validation accuracy %f\" % validation_accuracy)\n",
    "        print(\"Time elapsed\", (time.time()-t_i)/60.0, \"minutes\")\n",
    "train_writer.flush()\n",
    "validation_writer.flush()\n",
    "test_acc = test()\n",
    "print(\"Testing set accuracy %f\" % test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/matplotlib/ticker.py:2039: UserWarning: Data has no positive values, and therefore cannot be log-scaled.\n",
      "  \"Data has no positive values, and therefore cannot be \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHfNJREFUeJzt3XucVXW9//HXWwTGQFEZfiaOCYilkNyPecwbWgYS0lUw\nDwn1y+h3PGhWakd/Py21k1odw0tqKaJRamr+JENFIZTUaDBveEkkylEUHAQvKIh9zh9rDW7GPTN7\nwexZe8P7+Xjsx+z93Wuv/Vlr9uz3fL9r7+9SRGBmZlaq7fIuwMzMqouDw8zMMnFwmJlZJg4OMzPL\nxMFhZmaZODjMzCwTB4d1CElXSPq/JS57raTzyl1TOUkKSf3T6yVvezs8b9XvO6t8Dg4rStJ3Jc1u\n1vZsC20T2lpfREyJiHPbqbaNb8qb8VhJOknSY5LWSnpJ0h9K2YbN1V7bLulwSQ3tUVOJz7fZ+9m2\nbg4Oa8l9wEGSOgFI2h3oDAxt1tY/XbZaTANOAb4F9AT2AM4CRhVbOA0a/53kQNL2eddgxfkPwlry\nZ5KgGJLePgSYBzzTrO25iHgRQNK+kuZIWiXpGUnHNq2s+RCKpNMkLZf0oqT/XeS/210k3SHpdUl/\nkrR3+rimkHpU0huSxkuqlfQ7SavT576/2Ju9pA8D/weYEBFzIuKtiHg3IhZExKSC5f4g6XxJfwTW\nAv0kTZb0VFrPUklfb7bu7xRsz1ea3dd82z8t6ZG03gckDSq4b5mkb6c9ojWSbpRUI6kbMBvonW73\nG5J6t/C7q01/D69Lmi9pr3Tdl0n6cbPabpf0zRbWU5SkvSXNldQo6RVJMyXtXLAfbmm2/DRJP02v\n95B0dbqvXpB0XsE/IpMk/VHSf0tqBM7JUpd1oIjwxZeiF5Kg+GZ6/VLgK8D5zdquSa93A54HJgPb\nA0OBV4AB6f3XAuel10cBLwEDgQ8AvwQC6F+wbCNwQLqumcANBXVtXDa9/V/AFSRB15kk0FRke6YA\ny0rY7j8A/0jr2z5d5xhgb0DAYSSBMqxge14GPpruh18V2Z6mbR8KrAA+BnQCTgCWAV3T+5cBC4He\nwK7AU8CU9L7DgYY2ar8WeB04FOgK/BRYkN53APAisF16uzbdjt1aWNcm+7mgvT/wyXT9vUh6nBen\n9+0OvAnsnN7ePt3e4ent3wJXpvvpf6Xb+vX0vknABuA/0sftkPffgC/FL+5xWGvmk7wBQfJmfH96\nKWybn17/NMmb8vSI2BARfwFuAb5YZL3HAtMjYnFErKX4f5a/jYiFEbGBJDiGFFmmyTskb1h7RcQ7\nEXF/pO9EzdSSBNZGkhrS//zfbvrPPHVtWt+GdJ13RMRzkZgP3J1uf+H2PBERb7awPU1OBK6MiD9F\n0tuZAawDDixYZlpEvBgRq4BZbWx7MXdExH0RsQ44E/hXSXtGxEJgDXBkutwE4A8R8XKWlUfEkkh6\nbOsiYiXwE5IwJSKWkwRJ0+99FPBKRCyStBtwNHBKRLwZESuA/07raPJiRFyS7ve3Mm63dRAHh7Xm\nPuBgSbsCvSLiWeABkmMfu5L8h900dLQX8LH0TXi1pNXA8cAHi6y3N0nvpMnzRZYpfINfC3Rvpc6L\ngCXA3ekw0hktLNdIEjAbRUQdSaB0JelNFK1J0mhJD6VDYatJ3gBrW9iev7dS617At5rtpz3TdTTJ\nsu3FbKwlIt4AVhWsfwbwb+n1fwOuz7huJO0m6YZ0qOk1kh5jbcEiLT3HXiS9t+UF234lSc/jfbVb\n5XJwWGseBHoAXwP+CBARr5EMd3yN5L/Dv6XLPg/Mj4idCy7dI+IbRda7HKgruL3nlhQZEa9HxLci\noh9wDHCqpCOLLDoXqJM0opTVNl2R1JWk9/QjkmGdnYHf817QLG+2DR9qZb3PA+c3208fiIhfZ6mp\nDRtrkdSdZMjrxbTpl8A4SYOB/YDbSlxnoR+ktewfETuRhENh6N4GDJL0UZKe6My0/XmS3lVtwbbv\nFBEDCx7r6bqrgIPDWpQOFdQDp5IMUTVZkLYVfprqd8CHJU2U1Dm9/Iuk/Yqs+iZgsqT9JH0AyPod\nh5eBfk030oPN/SWJZCjmXeCfRbbnGZL/cG+Q9ElJO6QHZg9q4/m6kPRIVgIbJI0Gjmq2PZMkDUi3\n5+xW1vVzYIqkjynRTdIYSTu2udXJdveU1KON5Y6WdLCkLsC5wEMR8TxARDSQfPDheuCWEoaDuqQH\n55sunYAdgTeANZL2AL5T+ICIeBu4meRYz8KI+EfavpxkiO/HknaStF16oP2wErbdKoiDw9oyn2Qo\nYUFB2/1p28bgiIjXSd5MJ5D8d/sScAHJG+4mImI2ycdi55EMMT2U3rWuxJrOAWakwx3HAvsA95C8\nmT0IXB4R81p47L+nz/0TkiGcBpI31/EkB8TfJ922qSQB8SrwJeD2ZttzMUmPZkn6s6iIqCfprV2a\nrmsJyUHhNkXE08CvgaXptrf0qapfkYTXKmA47w0bNZkB7E9pw1SLgbcKLpOB7wHDSEL6DuDWIo9r\n6Tm+TBLET5Js/800Gz60yqfixxDNOk7aK3mC5JNFG/KuZ2sn6VCSIau9WvgQQXs8x4eAp4EPpsOb\nthVxj8NyIemzkrpK2oWkZzLLoVF+kjoDJwO/KGNobEcylHmDQ2Pr5OCwvHyd5PP9z5Eckyh2EN3a\nUdqzW00yNHRxmZ6jG/Aayfc8WjvWY1XMQ1VmZpaJexxmZpZJVU8iVltbG3369Mm7DDOzqrJo0aJX\nIqLX5j6+qoOjT58+1NfX512GmVlVkdTa7AZt8lCVmZll4uAwM7NMHBxmZpZJVR/jMDNryTvvvEND\nQwNvv/123qXkpqamhrq6Ojp37tyu63VwmNlWqaGhgR133JE+ffqQzH+5bYkIGhsbaWhooG/fvu26\n7ooZqpLULz2l5M1512Jm1e/tt9+mZ8+e22RoAEiiZ8+eZelxlTU4JF0jaYWkJ5q1j1JyTuolTSfd\niYilEfHVctZjZtuWbTU0mpRr+8vd47iW5NSRG6Xz+V8GjAYGAMdJGlDmOszM2jT+ygcZf+WDeZdR\n8coaHBFxH8k5AQodACxJexjrgRuAcaWuU9KJkuol1a9cubIdqzUzqw6NjY2MHDmS7t27c9JJJ3X4\n8+dxjGMPNj2vcAOwh6Sekq4Ahkr6bksPjoirImJERIzo1WuzvzFvZla1ampqOPfcc/nRj36Uy/NX\nzMHxiGiMiCkRsXdE/Ffe9ZiZbanrrruOQYMGMXjwYCZOnMiyZcs44ogjGDRoEEceeST/+Edy0slJ\nkyYxdepUDjroIPr168fNNyefEZowYQJ33HHHxvVNmjSJm2++mW7dunHwwQdTU1OTy3bl8XHcF4A9\nC27XpW1mZmXxvVmLefLFts8p9eTyZJlSjnMM6L0TZ48d2OL9ixcv5rzzzuOBBx6gtraWVatWccIJ\nJ2y8XHPNNUydOpXbbrsNgOXLl7NgwQKefvppjjnmGL7whS8wfvx4brrpJsaMGcP69eu59957+dnP\nflbiVpdPHj2OPwP7SOorqQvJOapvb+MxZmZVZe7cuXzxi1+ktrYWgF133ZUHH3yQL33pSwBMnDiR\nBQsWbFz+M5/5DNtttx0DBgzg5ZdfBmD06NHMmzePdevWMXv2bA499FB22GGHjt+YZsra45D0a+Bw\noFZSA3B2RFwt6STgLqATcE1ELC5nHWa2bWutZ1Coqadx49f/tZzlFNW1a9eN15tOsFdTU8Phhx/O\nXXfdxY033siECRM6vK5iyv2pquMiYveI6BwRdRFxddr++4j4cHo84/xy1mBmlocjjjiC3/zmNzQ2\nNgKwatUqDjroIG644QYAZs6cySGHHNLmesaPH8/06dO5//77GTVqVJvLdwRPOWJmVgYDBw7kzDPP\n5LDDDqNTp04MHTqUSy65hMmTJ3PRRRfRq1cvpk+f3uZ6jjrqKCZOnMi4cePo0qXLxvY+ffrw2muv\nsX79em677TbuvvtuBgzomK/EOTjMzMqk6UB4oblz575vuWuvvXaT22+88cbG6507d2bVquZfh4Nl\ny5a1S42boyqDQ9JYYGz//v3zLsXMtiJ5HNuoRhXzPY4sImJWRJzYo0ePvEsxM9vmVGVwmJlZfhwc\nZmaWiYPDzMwycXCYmTWZPia5WKscHGZmVWbOnDkMHz6c/fffn+HDhxf9iG85VeXHcc3MtmW1tbXM\nmjWL3r1788QTT/CpT32KF17ouLli3eMwMyuTck2rPnToUHr37g0k31B/6623WLduXYdtl3scZrb1\nm30GvPR428u99Fjys5TjHB/cH0b/sMW7O2pa9VtuuYVhw4ZtMkliuVVlj0PSWElXrVmzJu9SzMyK\n6ohp1RcvXszpp5/OlVde2YFbVqU9joiYBcwaMWLE1/KuxcyqQCs9g0009TQm39H6cmWQdVr1hoYG\nPvvZz3Ldddex9957d2itVdnjMDOrdOWcVn316tWMGTOGH/7wh3z84x8v30a0wMFhZlYGhdOqDx48\nmFNPPZVLLrmE6dOnM2jQIK6//np++tOftrmeo446ivnz5/OJT3xi47Tql156KUuWLOH73/8+Q4YM\nYciQIaxYsaLcm7SRmrpE1WjEiBFRX1+fdxlmVoGeeuop9ttvv2wPynGoqlyK7QdJiyJixOausyqP\ncZiZlcVWFBjl5KEqMzPLxMFhZmaZODjMzCwTB4eZmWXi4DAzS02+czKT75ycdxkVz8FhZlYm06ZN\nY7/99uP4448vev+ZZ57JnnvuSffu3Tu4si1TlcHhuarMrBpcfvnlzJkzh5kzZxa9f+zYsSxcuLCD\nq9pyVRkcETErIk7s0aNH3qWYmRU1ZcoUli5dyujRozn33HOZPHky+++/P4MGDeKWW24B4MADD2T3\n3XfPudLs/AVAM9vqXbDwAp5e9XSbyzUtU8pxjn133ZfTDzi9xfuvuOIK7rzzTubNm8dFF11Ejx49\nePzxZGr3V199tcTKK5ODw8yszO65556NkxsC7LLLLjlWs+UcHGa21WutZ1CoqacxfdT0cpZT9ary\nGIeZWTX55Cc/yWWXXbbxdrUPVTk4zMzK7KyzzuLVV1/lox/9KIMHD2bevHkAnHbaadTV1bF27Vrq\n6uo455xz8i20RB6qMjMrk2XLlm28PmPGjPfdf+GFF3LhhRd2YEXtw8FhZpbysY3SeKjKzMwycXCY\nmVkmDg4zM8vEwWFmZplUZXB4kkMzK4e/T/wyf5/45bzLqHhVGRye5NDMqkFr06qvXbuWMWPGsO++\n+zJw4EDOOOOMHCrcPP44rplZmVx++eXcc8891NXVFb3/29/+NiNHjmT9+vUceeSRzJ49m9GjR3dw\nldk5OMzMyqBwWvVjjz2WpUuXUl9fjyTOPvtsPv/5zzNy5EgAunTpwrBhw2hoaMi56tI4OMxsq/fS\nD37Auqfanlb97aeTZUo5ztF1v3354H/+Z4v3Z5lWffXq1cyaNYuTTz65zeetBA4OM7Mya21a9Q0b\nNnDccccxdepU+vXrl0d5mTk4zGyr11rPoFBTT2Ov668rZzmbOPHEE9lnn3045ZRTOuw5t1RVfqrK\nzKyatDSt+llnncWaNWu4+OKL8yptszg4zMzKrNi06g0NDZx//vk8+eSTDBs2jCFDhvCLX/wi71JL\n4qEqM7MyaWta9YjowGraj4PDzCzVkcc2qpmHqszMLBMHh5lttap1KKi9lGv7HRxmtlWqqamhsbFx\nmw2PiKCxsZGampp2X7ePcZjZVqmuro6GhgZWrlyZdym5qampaXGerC3h4DCzrVLnzp3p27dv3mVs\nlapyqMrn4zAzy09VBofPx2Fmlp+qDA4zM8uPg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweH\nmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxm\nZpZJVQaHTx1rZpafkoJD0hdLaesoPnWsmVl+Su1xfLfENjMz28pt39qdkkYDRwN7SJpWcNdOwIZy\nFmZmZpWp1eAAXgTqgWOARQXtrwPfLFdRZmZWuVoNjoh4FHhU0q8i4p0OqsnMzCpYWz2OJgdIOgfY\nK32MgIiIfuUqzMzMKlOpwXE1ydDUIuDd8pVjZmaVrtTgWBMRs8taiZmZVYVSg2OepIuAW4F1TY0R\n8XBZqjIzs4pVanB8LP05oqAtgCPatxwzM6t0JQVHRIwsdyFmZlYdSp1yZDdJV0uand4eIOmr5S3N\nzMwqUalTjlwL3AX0Tm//FTilHAWZmVllKzU4aiPiJuCfABGxAX8s18xsm1RqcLwpqSfJAXEkHQh4\nTnMzs21QqZ+qOhW4Hdhb0h+BXsAXylaVmZlVrFI/VfWwpMOAj5BMN/KM564yM9s2tTWt+hERMVfS\n55rd9WFJRMStZazNzMwqUFs9jsOAucDYIvcFyTfJzcxsG9LWtOpnpz8nd0w5ZmZW6doaqjq1tfsj\n4iftW46ZmVW6toaqdkx/fgT4F5JPVkEydLWwXEWZmVnlamuo6nsAku4DhkXE6+ntc4A7yl6dmZlV\nnFK/ALgbsL7g9vq0zczMtjGlfgHwOmChpN+mtz8DzChPSW2TNBYY279//7xKMDPbZikiSltQGg4c\nnN68LyL+UraqSjRixIior6/Puwwzs6oiaVFEjGh7yeJK7XEQEYskPQ/UpE/8oYj4x+Y+sZmZVadS\nz8dxjKRngb8B89OfPge5mdk2qNSD4+cCBwJ/jYi+wCeAh8pWlZmZVaxSg+OdiGgEtpO0XUTMY9Pz\nj5uZ2Tai1GMcqyV1B+4DZkpaAbxZvrLMzKxSldrjGAesBb4J3Ak8R/GJD83MbCvXZo9DUifgdxEx\nkuTUsbl9f8PMzPLXZo8jIt4F/impRwfUY2ZmFa7UYxxvAI9LmkPBsY2ImFqWqszMrGKVGhy38t5J\nm5q+aq72L8fMzCpdW+fjGAfURcRl6e2FQC+S8Di9/OWZmVmlaesYx2m8dw4OgC7AcOBwYEqZajIz\nswrW1lBVl4h4vuD2gohYBayS1K2MdZmZWYVqq8exS+GNiDip4Gav9i/HzMwqXVvB8SdJX2veKOnr\n+NSxZmbbpLaGqr4J3CbpS8DDadtwoCvJyZzMzGwb09Y5x1cAB0k6AhiYNt8REXPLXpmZmVWkkr7H\nkQaFw8LMzEqe5NDMzAxwcJiZWUYODjMzy8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmZpaJg8PM\nzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweHmZllUpXBIWmspKvWrFmTdylmZtucqgyOiJgVESf2\n6NEj71LMzLY5VRkcZmaWHweHmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4\nzMwsEweHmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweHmZll4uAw\nM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmZpaJg8PM\nzDJxcJiZWSYODjMzy8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMz\ny8TBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMzy8TBYWZmmTg4zMws\nEweHmZll4uAwM7NMHBxmZpaJg8PMzDJxcJiZWSYODjMzy2T7vAtoIqkbcDmwHvhDRMzMuSQzMyui\nrD0OSddIWiHpiWbtoyQ9I2mJpDPS5s8BN0fE14BjylmXmZltvnIPVV0LjCpskNQJuAwYDQwAjpM0\nAKgDnk8Xe7fMdZmZ2WYqa3BExH3AqmbNBwBLImJpRKwHbgDGAQ0k4dFqXZJOlFQvqX7lypXlKNvM\nzFqRx8HxPXivZwFJYOwB3Ap8XtLPgFktPTgiroqIERExolevXuWt1MzM3qdiDo5HxJvA5LzrMDOz\n1uXR43gB2LPgdl3aZmZmVSCP4PgzsI+kvpK6ABOA23Oow8zMNkO5P477a+BB4COSGiR9NSI2ACcB\ndwFPATdFxOJy1mFmZu2nrMc4IuK4Ftp/D/y+nM9tZmbl4SlHzMwsEweHmZllUpXBIWmspKvWrFmT\ndylmZtscRUTeNWw2Sa8Dz+RdRwlqgVfyLqIErrN9VUOd1VAjuM729pGI2HFzH1wxXwDcTM9ExIi8\ni2iLpHrX2X5cZ/uphhrBdbY3SfVb8viqHKoyM7P8ODjMzCyTag+Oq/IuoESus325zvZTDTWC62xv\nW1RnVR8cNzOzjlftPQ4zM+tgDg4zM8ukaoOjhfOW507SnpLmSXpS0mJJJ6ft50h6QdIj6eXonOtc\nJunxtJb6tG1XSXMkPZv+3CXnGj9SsL8ekfSapFMqYV9KukbSCklPFLQV3X9KTEtfq49JGpZznRdJ\nejqt5beSdk7b+0h6q2C/XpFznS3+niV9N92fz0j6VM513lhQ4zJJj6TtuezPVt6D2u/1GRFVdwE6\nAc8B/YAuwKPAgLzrSmvbHRiWXt8R+CvJudXPAb6dd30FdS4Dapu1XQickV4/A7gg7zqb/c5fAvaq\nhH0JHAoMA55oa/8BRwOzAQEHAn/Kuc6jgO3T6xcU1NmncLkK2J9Ff8/p39OjQFegb/pe0CmvOpvd\n/2Pg/+W5P1t5D2q312e19jhaOm957iJieUQ8nF5/nWTq+D3yrapk44AZ6fUZwGdyrKW5I4HnIuLv\neRcCEBH3AauaNbe0/8YB10XiIWBnSbvnVWdE3B3J6Q0AHiI5mVquWtifLRkH3BAR6yLib8ASkveE\nsmutTkkCjgV+3RG1tKSV96B2e31Wa3C0dN7yiiKpDzAU+FPadFLaFbwm72EgIIC7JS2SdGLatltE\nLE+vvwTslk9pRU1g0z/IStqXTVraf5X8ev0KyX+bTfpK+ouk+ZIOyauoAsV+z5W6Pw8BXo6IZwva\nct2fzd6D2u31Wa3BUfEkdQduAU6JiNeAnwF7A0OA5SRd2jwdHBHDgNHAv0s6tPDOSPqwFfFZbSVn\nijwG+E3aVGn78n0qaf+1RNKZwAZgZtq0HPhQRAwFTgV+JWmnvOqjCn7PzRzHpv/c5Lo/i7wHbbSl\nr89qDY6KPm+5pM4kv7CZEXErQES8HBHvRsQ/gZ/TQV3rlkTEC+nPFcBv03pebuqipj9X5FfhJkYD\nD0fEy1B5+7JAS/uv4l6vkiYBnwaOT99ESId+GtPri0iOHXw4rxpb+T1X4v7cHvgccGNTW577s9h7\nEO34+qzW4KjY85an45xXA09FxE8K2gvHDD8LPNH8sR1FUjdJOzZdJzlY+gTJPjwhXewE4P/nU+H7\nbPKfXCXty2Za2n+3A19OP71yILCmYMigw0kaBZwGHBMRawvae0nqlF7vB+wDLM2nylZ/z7cDEyR1\nldSXpM6FHV1fM58Ano6IhqaGvPZnS+9BtOfrs6OP+LfjJweOJvm0wHPAmXnXU1DXwSRdwMeAR9LL\n0cD1wONp++3A7jnW2I/kUymPAoub9h/QE7gXeBa4B9i1AvZnN6AR6FHQlvu+JAmy5cA7JGPCX21p\n/5F8WuWy9LX6ODAi5zqXkIxpN70+r0iX/Xz6engEeBgYm3OdLf6egTPT/fkMMDrPOtP2a4EpzZbN\nZX+28h7Ubq9PTzliZmaZVOtQlZmZ5cTBYWZmmTg4zMwsEweHmZll4uAwM7NMHBxmRUh6V5vOzNtu\nMzCns6ZWyndPzDLbPu8CzCrUWxExJO8izCqRexxmGaTnW7hQyblMFkrqn7b3kTQ3nZDvXkkfStt3\nU3LOi0fTy0HpqjpJ+nl6voS7Je2QLj81PY/CY5JuyGkzzVrl4DArbodmQ1XjC+5bExH7A5cCF6dt\nlwAzImIQyaSB09L2acD8iBhMch6HxWn7PsBlETEQWE3yLWNIzpMwNF3PlHJtnNmW8DfHzYqQ9EZE\ndC/Svgw4IiKWphPJvRQRPSW9QjIlxjtp+/KIqJW0EqiLiHUF6+gDzImIfdLbpwOdI+I8SXcCbwC3\nAbdFxBtl3lSzzNzjMMsuWriexbqC6+/y3vHGMSTzBg0D/pzOumpWURwcZtmNL/j5YHr9AZJZmgGO\nB+5Pr98LfANAUidJPVpaqaTtgD0jYh5wOtADeF+vxyxv/m/GrLgdJD1ScPvOiGj6SO4ukh4j6TUc\nl7b9BzBd0neAlcDktP1k4CpJXyXpWXyDZHbVYjoBv0zDRcC0iFjdbltk1k58jMMsg/QYx4iIeCXv\nWszy4qEqMzPLxD0OMzPLxD0OMzPLxMFhZmaZODjMzCwTB4eZmWXi4DAzs0z+B+QqhHr4Z0EQAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb6b1812b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHZtJREFUeJzt3X28VVW97/HPFwS3iaICx9RtAooPkIBI6TEfAh8SuYhl\nCuol4Xgyu9fIvB219F4ttWva7ZaKT6UoRvmsRzJUFPLhpBl6NEWxlDC3oSAISiio/c4fc2xcbPfD\nmrDXnmuxv+/Xa72Ya8y55vzNsRfrt8YYc42piMDMzKxcXYoOwMzMaosTh5mZ5eLEYWZmuThxmJlZ\nLk4cZmaWixOHmZnl4sRh7UrSVZL+d9FxdCRJv5X0r2n5BEn3d9BxJ0p6tCOOZVbKicNykbRQ0ruS\nVkp6S9I9knZsXB8Rp0TE+UXGWErSoZLmSHpH0lJJT0s6U1JdJY4XEdMj4rD22JekkLRLe+yrjGOt\nTX5mbXHisPUxJiJ6ANsBbwCXFRxPsyQdA9wG/BLYKSJ6AeOAemDHFl6zScdFaI1c77XFicPWW0S8\nR/bBPLCxTNL1ki5Iy1tL+rWkJal18mtJ9SXbTpS0ILUG/iLphJJ1/yLphfS6+yTtlMol6f9LWizp\nbUnPSvp009gkCfgx8P2I+FlELEsxvxgR34iIP6ftzpN0m6RfSHobmCjps5Iek7Rc0iJJl0vqXrLv\nQyXNl7RC0uWAmpzToyXPd5c0S9IySS9KOrZJXU1JrbZ3JP1e0s5p3cNps2dS625cC38GpfhWpJgO\nToXHSHqyyYanS/r3FvbTIkm3Sno9HeNhSYNS+WckvSGpa8m2X5L0TFruIuksSS+n1t4tkrZJ6/qm\nFtVJkv4KzM4blxXHicPWm6RPkH2Df7yFTboAU4GdgE8B7wKXp9duDlwKjIqILYD9gKfTurHAd4Ev\nAX2AR4BfpX0eBhwI7Ar0BI4FljZz7N3IWha3l3EqY8kS4FbAdOBD4FtAb+CfgYOB/5Fi6w3cAZyT\n1r8MfK65naZznEXW4vknYDxwhaSBJZuNB74HbA28BFwIEBEHpvVDIqJHRNzcQuz7pBh6A+cCd6QP\n57uBfpL2KNl2AjCt7er4mJnAgHQOT5HVERHxB7K6L+2aKz3GN4CjgIOA7YG3gClN9n0QsAfwhfWI\ny4oSEX74UfYDWAisBJYD7wN/A/YsWX89cEELrx0KvJWWN0/7OBrYrMl2M4GTSp53AVaRJaCRwJ+A\nfYEurcS5PxBAXUnZTemYq4AJqew84OE2zvk04M60/BXg8ZJ1AhqAf03PJwKPpuVxwCNN9nU1cG5J\nXf28ZN0RwPyS5wHs0kpcE1P9q6TsiZJzuxK4MC0PIvvg3rSFff228RzaqIutUlw90/MzgelpeZtU\nt9ul5y8AB5e8drv0ntkE6Jv207/o97Qf+R9ucdj6OCoitgLqgFOBhyR9sulGkj4h6WpJr6RuoIeB\nrSR1jYi/k32wngIsSt01u6eX7gT8NHUVLQeWkX1A7xARs8laLVOAxZKukbRlMzE2tkK2ayyIiPEp\n7qeAriXbvtok7l1Tt9rrKe4fkH2jh+yb89rtI/tEXOf1JXYC9mk8j3QuJwCldfV6yfIqoEcL+2rJ\naymGRq+kGAFuAI5P3XYTgFsiYnWenUvqKumi1N30NtkXB/ioPn4BjEmtq2PJEuWitG4n4M6Sc3+B\nrDW3bckhWqo7q2JOHLbeIuLDiLiD7MNg/2Y2+V9kXUb7RMSWZF1MkMYEIuK+iDiU7MN9PvCztP5V\n4GsRsVXJY7OI+F163aURsTfZ2MquwL81c+wXgdfIurvaPJUmz69M8QxIcX+Xj8YxFlEysJ4+lJsd\naE/n8VCT8+gREV8vI6Zy7ZBiaPQpslYIEfE4sAY4ADgeuHE99n88WVfeIWRdg31TeePf8DXgMbJ6\nntDkGK+SdUWWnn9dek0jT89dg5w4bL2lgeqxZP3zLzSzyRZk4xrLU7/7uSWv3VbS2PRNdTVZ99c/\n0uqrgO+UDML2VHaFVOOA7D6SugF/B94red1aEfEPssR1rqSvKhuol6QBrPuNtzlbAG8DK1MrqPSD\n/h5gUBoE3gSYzLotiFK/BnaVNEFSt/T4TJNxh9a8AfRvY5t/AianfR9DNl7wm5L108haaO9HRFu/\n+dhEUl3JoxtZXawma8F9gqz11dQ04AxgT7Lxn0ZXARfqowsb+qT3i9U4Jw5bHzMkrST7cL0QODEi\n5jWz3U+AzYA3yQbQ7y1Z1wU4nezb8TKyQdKvA0TEncAPgZtS98hzwKj0ui3JWiZvkXXLLAUuaS7I\nyAaUjwX+O9m33zeBW4BrgFtbOb9vk33Tficda+3AdES8CRwDXJSOPQD4jxaO/w7ZwPH4dJ6vp/Pa\ntJVjlzoPuCF19Rzbwja/TzG8Sfa3+HJElF4scCPwabIupbZcSZboGx9TyZLCK2Stt+dp/kKIO0nd\nUhGxqqT8p2SD9PdLeie9dp8y4rAqp3W7R81sYyJpM2AxMCzSJcgVOs7LZN2LD1TqGFY93OIw27h9\nHfhDhZPG0WRjFf4tRifhX2uabaQkLSQbxD6qgsf4LdlFChPSuJJ1Au6qMjOzXNxVZWZmudR0V1Xv\n3r2jb9++RYdhZlZTnnzyyTcjos/6vr6mE0ffvn2ZO3du0WGYmdUUSa9syOvdVWVmZrk4cZiZWS5O\nHGZmlktNj3GYmbXk/fffp6Ghgffee6/oUApTV1dHfX093bp1a9f9OnGY2UapoaGBLbbYgr59+7Lu\nBMKdQ0SwdOlSGhoa6NevX7vuu2q6qiT1l3StpNuKjsXMat97771Hr169OmXSAJBEr169KtLiqmji\nkHSdsntDP9ek/HBl919+SdJZABGxICJOqmQ8Zta5dNak0ahS51/pFsf1wOGlBenG9lPIpskeCBzX\n5B7MZmaFGHf1Y4y7+rGiw6h6FU0cEfEw2b0WSn0WeCm1MNaQ3Qe67Ju7SDpZ0lxJc5csWdKO0ZqZ\n1YalS5cyYsQIevTowamnntrhxy9ijGMH1r3PcAPZ7S97SboK2EvSd1p6cURcExHDI2J4nz7r/Yt5\nM7OaVVdXx/nnn8+PfvSjQo5fNYPjEbE0Ik6JiJ0j4v8WHY+Z2YaaNm0agwcPZsiQIUyYMIGFCxcy\ncuRIBg8ezMEHH8xf//pXACZOnMjkyZPZb7/96N+/P7fdll0jNH78eO655561+5s4cSK33XYbm2++\nOfvvvz91dXWFnFcRl+O+BuxY8rw+lZmZVcT3Zszj+b+93eZ2zy/KtilnnGPg9lty7phBLa6fN28e\nF1xwAb/73e/o3bs3y5Yt48QTT1z7uO6665g8eTJ33XUXAIsWLeLRRx9l/vz5HHnkkXz5y19m3Lhx\n3HLLLYwePZo1a9bw4IMPcuWVV5Z51pVTRIvjD8AASf0kdSe7H/PdBcRhZlYxs2fP5phjjqF3794A\nbLPNNjz22GMcf/zxAEyYMIFHH3107fZHHXUUXbp0YeDAgbzxxhsAjBo1ijlz5rB69WpmzpzJgQce\nyGabbdbxJ9NERVsckn4FfB7oLakBODcirpV0KnAf0BW4LiLmVTIOM+vcWmsZlGpsadz8tX+uZDjN\n2nTTTdcuN95gr66ujs9//vPcd9993HzzzYwfP77D42pOpa+qOi4itouIbhFRHxHXpvLfRMSuaTzj\nwkrGYGZWhJEjR3LrrbeydOlSAJYtW8Z+++3HTTfdBMD06dM54IAD2tzPuHHjmDp1Ko888giHH354\nm9t3BE85YmZWAYMGDeLss8/moIMOomvXruy1115cdtllTJo0iUsuuYQ+ffowderUNvdz2GGHMWHC\nBMaOHUv37t3Xlvft25e3336bNWvWcNddd3H//fczcGDH/CTOicPMrEIaB8JLzZ49+2PbXX/99es8\nX7ly5drlbt26sWxZ05/DwcKFC9slxvVRk4lD0hhgzC677FJ0KGa2ESlibKMWVc3vOPKIiBkRcXLP\nnj2LDsXMrNOpycRhZmbFceIwM7NcnDjMzCwXJw4zs0ZTR2cPa5UTh5lZjZk1axZ77703e+65J3vv\nvXezl/hWUk1ejmtm1pn17t2bGTNmsP322/Pcc8/xhS98gdde67i5Yt3iMDOrkEpNq77XXnux/fbb\nA9kv1N99911Wr17dYeflFoeZbfxmngWvP9v2dq//Mfu3nHGOT+4Joy5qcXVHTat+++23M2zYsHUm\nSay0mmxxSBoj6ZoVK1YUHYqZWbM6Ylr1efPmceaZZ3L11Vd34JnVaIsjImYAM4YPH/7VomMxsxrQ\nSstgHY0tjUn3tL5dBeSdVr2hoYEvfvGLTJs2jZ133rlDY63JFoeZWbWr5LTqy5cvZ/To0Vx00UV8\n7nOfq9xJtMCJw8ysAkqnVR8yZAinn346l112GVOnTmXw4MHceOON/PSnP21zP4cddhgPPfQQhxxy\nyNpp1S+//HJeeuklvv/97zN06FCGDh3K4sWLK31Ka6mxSVSLhg8fHnPnzi06DDOrQi+88AJ77LFH\nvhcV2FVVKc3Vg6QnI2L4+u6zJsc4zMwqYiNKGJXkriozM8vFicPMzHJx4jAzs1ycOMzMLBcnDjOz\nZNK9k5h076Siw6h6ThxmZhVy6aWXsscee3DCCSc0u/7ss89mxx13pEePHh0c2YapycThuarMrBZc\nccUVzJo1i+nTpze7fsyYMTzxxBMdHNWGq8nEEREzIuLknj17Fh2KmVmzTjnlFBYsWMCoUaM4//zz\nmTRpEnvuuSeDBw/m9ttvB2Dfffdlu+22KzjS/PwDQDPb6P3wiR8yf9n8Nrdr3KaccY7dt9mdMz97\nZovrr7rqKu69917mzJnDJZdcQs+ePXn22Wxq97feeqvMyKuTE4eZWYU98MADayc3BNh6660LjGbD\nOXGY2UavtZZBqcaWxtTDp1YynJpXk2McZma15NBDD2XKlClrn9d6V5UTh5lZhZ1zzjm89dZbfPrT\nn2bIkCHMmTMHgDPOOIP6+npWrVpFfX095513XrGBlsldVWZmFbJw4cK1yzfccMPH1l988cVcfPHF\nHRhR+3DiMDNLPLZRHndVmZlZLk4cZmaWixOHmZnl4sRhZma51GTi8CSHZlYJr0z4Cq9M+ErRYVS9\nmkwcnuTQzGpBa9Oqr1q1itGjR7P77rszaNAgzjrrrAIiXD++HNfMrEKuuOIKHnjgAerr65td/+1v\nf5sRI0awZs0aDj74YGbOnMmoUaM6OMr8nDjMzCqgdFr1Y489lgULFjB37lwkce6553L00UczYsQI\nALp3786wYcNoaGgoOOryOHGY2Ubv9R/8gNUvtD2t+nvzs23KGefYdI/d+eR3v9vi+jzTqi9fvpwZ\nM2bwzW9+s83jVgMnDjOzCmttWvUPPviA4447jsmTJ9O/f/8iwsvNicPMNnqttQxKNbY0drpxWiXD\nWcfJJ5/MgAEDOO200zrsmBuqJq+qMjOrJS1Nq37OOeewYsUKfvKTnxQV2npx4jAzq7DmplVvaGjg\nwgsv5Pnnn2fYsGEMHTqUn//850WHWhZ3VZmZVUhb06pHRAdG036cOMzMko4c26hl7qoyM7NcnDjM\nbKNVq11B7aVS5+/EYWYbpbq6OpYuXdppk0dEsHTpUurq6tp93x7jMLONUn19PQ0NDSxZsqToUApT\nV1fX4jxZG8KJw8w2St26daNfv35Fh7FRqsmuKt+Pw8ysODWZOHw/DjOz4tRk4jAzs+I4cZiZWS5O\nHGZmlosTh5mZ5eLEYWZmuThxmJlZLk4cZmaWixOHmZnl4sRhZma5OHGYmVkuThxmZpaLE4eZmeXi\nxGFmZrk4cZiZWS5OHGZmlosTh5mZ5eLEYWZmudRk4vCtY83MilNW4pB0TDllHcW3jjUzK065LY7v\nlFlmZmYbuU1aWylpFHAEsIOkS0tWbQl8UMnAzMysOrWaOIC/AXOBI4EnS8rfAb5VqaDMzKx6tZo4\nIuIZ4BlJv4yI9zsoJjMzq2JttTgafVbSecBO6TUCIiL6VyowMzOrTuUmjmvJuqaeBD6sXDhmZlbt\nyk0cKyJiZkUjMTOzmlBu4pgj6RLgDmB1Y2FEPFWRqMzMrGqVmzj2Sf8OLykLYGT7hmNmZtWurMQR\nESMqHYiZmdWGcqcc2VbStZJmpucDJZ1U2dDMzKwalTvlyPXAfcD26fmfgNMqEZCZmVW3chNH74i4\nBfgHQER8gC/LNTPrlMpNHH+X1ItsQBxJ+wKe09zMrBMq96qq04G7gZ0l/QfQB/hyxaIyM7OqVe5V\nVU9JOgjYjWy6kRc9d5WZWefU1rTqIyNitqQvNVm1qyQi4o4KxmZmZlWorRbHQcBsYEwz64Lsl+Rm\nZtaJtDWt+rnp30kdE46ZmVW7trqqTm9tfUT8uH3DMTOzatdWV9UW6d/dgM+QXVkFWdfVE5UKyszM\nqldbXVXfA5D0MDAsIt5Jz88D7ql4dGZmVnXK/QHgtsCakudrUpmZmXUy5f4AcBrwhKQ70/OjgBsq\nE1LbJI0Bxuyyyy5FhWBm1mkpIsrbUNob2D89fTgi/rNiUZVp+PDhMXfu3KLDMDOrKZKejIjhbW/Z\nvHJbHETEk5JeBerSgT8VEX9d3wObmVltKvd+HEdK+jPwF+Ch9K/vQW5m1gmVOzh+PrAv8KeI6Acc\nAjxesajMzKxqlZs43o+IpUAXSV0iYg7r3n/czMw6iXLHOJZL6gE8DEyXtBj4e+XCMjOzalVui2Ms\nsAr4FnAv8DLNT3xoZmYbuTZbHJK6Ar+OiBFkt44t7PcbZmZWvDZbHBHxIfAPST07IB4zM6ty5Y5x\nrASelTSLkrGNiJhckajMzKxqlZs47uCjmzY1/tRc7R+OmZlVu7buxzEWqI+IKen5E0AfsuRxZuXD\nMzOzatPWGMcZfHQPDoDuwN7A54FTKhSTmZlVsba6qrpHxKslzx+NiGXAMkmbVzAuMzOrUm21OLYu\nfRIRp5Y87dP+4ZiZWbVrK3H8XtJXmxZK+hq+dayZWafUVlfVt4C7JB0PPJXK9gY2JbuZk5mZdTJt\n3XN8MbCfpJHAoFR8T0TMrnhkZmZWlcr6HUdKFE4WZmZW9iSHZmZmgBOHmZnl5MRhZma5OHGYmVku\nThxmZpaLE4eZmeXixGFmZrk4cZiZWS5OHGZmlosTh5mZ5eLEYWZmuThxmJlZLk4cZmaWixOHmZnl\nUpOJQ9IYSdesWLGi6FDMzDqdmkwcETEjIk7u2bNn0aGYmXU6NZk4zMysOE4cZmaWixOHmZnl4sRh\nZma5OHGYmVkuThxmZpaLE4eZmeXixGFmZrk4cZiZWS5OHGZmlosTh5mZ5eLEYWZmuThxmJlZLk4c\nZmaWixOHmZnl4sRhZma5OHGYmVkuThxmZpaLE4eZmeXixGFmZrk4cZiZWS5OHGZmlosTh5mZ5eLE\nYWZmuThxmJlZLk4cZmaWixOHmZnl4sRhZma5OHGYmVkuThxmZpaLE4eZmeXixGFmZrk4cZiZWS5O\nHGZmlosTh5mZ5eLEYWZmuThxmJlZLk4cZmaWixOHmZnl4sRhZma5OHGYmVkuThxmZpaLE4eZmeXi\nxGFmZrk4cZiZWS5OHGZmlosTh5mZ5eLEYWZmuThxmJlZLk4cZmaWixOHmZnl4sRhZma5OHGYmVku\nThxmZpaLE4eZmeWySdEBNJK0OXAFsAb4bURMLzgkMzNrRkVbHJKuk7RY0nNNyg+X9KKklySdlYq/\nBNwWEV8FjqxkXGZmtv4q3VV1PXB4aYGkrsAUYBQwEDhO0kCgHng1bfZhheMyM7P1VNHEEREPA8ua\nFH8WeCkiFkTEGuAmYCzQQJY8Wo1L0smS5kqau2TJkkqEbWZmrShicHwHPmpZQJYwdgDuAI6WdCUw\no6UXR8Q1ETE8Iob36dOnspGamdnHVM3geET8HZhUdBxmZta6IlocrwE7ljyvT2VmZlYDikgcfwAG\nSOonqTswHri7gDjMzGw9VPpy3F8BjwG7SWqQdFJEfACcCtwHvADcEhHzKhmHmZm1n4qOcUTEcS2U\n/wb4TSWPbWZmleEpR8zMLBcnDjMzy6UmE4ekMZKuWbFiRdGhmJl1OoqIomNYb5LeAV4sOo4y9Abe\nLDqIMjjO9lULcdZCjOA429tuEbHF+r64an4AuJ5ejIjhRQfRFklzHWf7cZztpxZiBMfZ3iTN3ZDX\n12RXlZmZFceJw8zMcqn1xHFN0QGUyXG2L8fZfmohRnCc7W2D4qzpwXEzM+t4td7iMDOzDubEYWZm\nudRs4mjhvuWFk7SjpDmSnpc0T9I3U/l5kl6T9HR6HFFwnAslPZtimZvKtpE0S9Kf079bFxzjbiX1\n9bSktyWdVg11Kek6SYslPVdS1mz9KXNpeq/+UdKwguO8RNL8FMudkrZK5X0lvVtSr1cVHGeLf2dJ\n30n1+aKkLxQc580lMS6U9HQqL6Q+W/kMar/3Z0TU3APoCrwM9Ae6A88AA4uOK8W2HTAsLW8B/Ins\n3urnAd8uOr6SOBcCvZuUXQyclZbPAn5YdJxN/uavAztVQ10CBwLDgOfaqj/gCGAmIGBf4PcFx3kY\nsEla/mFJnH1Lt6uC+mz275z+Pz0DbAr0S58FXYuKs8n6/wf8nyLrs5XPoHZ7f9Zqi6Ol+5YXLiIW\nRcRTafkdsqnjdyg2qrKNBW5IyzcARxUYS1MHAy9HxCtFBwIQEQ8Dy5oUt1R/Y4FpkXkc2ErSdkXF\nGRH3R3Z7A4DHyW6mVqgW6rMlY4GbImJ1RPwFeInsM6HiWotTkoBjgV91RCwtaeUzqN3en7WaOFq6\nb3lVkdQX2Av4fSo6NTUFryu6GwgI4H5JT0o6OZVtGxGL0vLrwLbFhNas8az7H7Ka6rJRS/VXze/X\nfyH7ttmon6T/lPSQpAOKCqpEc3/naq3PA4A3IuLPJWWF1meTz6B2e3/WauKoepJ6ALcDp0XE28CV\nwM7AUGARWZO2SPtHxDBgFPA/JR1YujKyNmxVXKut7E6RRwK3pqJqq8uPqab6a4mks4EPgOmpaBHw\nqYjYCzgd+KWkLYuKjxr4OzdxHOt+uSm0Ppv5DFprQ9+ftZo4qvq+5ZK6kf3BpkfEHQAR8UZEfBgR\n/wB+Rgc1rVsSEa+lfxcDd6Z43mhsoqZ/FxcX4TpGAU9FxBtQfXVZoqX6q7r3q6SJwH8DTkgfIqSu\nn6Vp+UmysYNdi4qxlb9zNdbnJsCXgJsby4qsz+Y+g2jH92etJo6qvW956ue8FnghIn5cUl7aZ/hF\n4Lmmr+0okjaXtEXjMtlg6XNkdXhi2uxE4N+LifBj1vkmV0112URL9Xc38JV09cq+wIqSLoMOJ+lw\n4AzgyIhYVVLeR1LXtNwfGAAsKCbKVv/OdwPjJW0qqR9ZnE90dHxNHALMj4iGxoKi6rOlzyDa8/3Z\n0SP+7XjlwBFkVwu8DJxddDwlce1P1gT8I/B0ehwB3Ag8m8rvBrYrMMb+ZFelPAPMa6w/oBfwIPBn\n4AFgmyqoz82BpUDPkrLC65IskS0C3ifrEz6ppfoju1plSnqvPgsMLzjOl8j6tBvfn1elbY9O74en\ngaeAMQXH2eLfGTg71eeLwKgi40zl1wOnNNm2kPps5TOo3d6fnnLEzMxyqdWuKjMzK4gTh5mZ5eLE\nYWZmuThxmJlZLk4cZmaWixOHWTMkfah1Z+ZttxmY06yp1fLbE7PcNik6ALMq9W5EDC06CLNq5BaH\nWQ7pfgsXK7uXyROSdknlfSXNThPyPSjpU6l8W2X3vHgmPfZLu+oq6Wfpfgn3S9osbT853Ufhj5Ju\nKug0zVrlxGHWvM2adFWNK1m3IiL2BC4HfpLKLgNuiIjBZJMGXprKLwUeioghZPdxmJfKBwBTImIQ\nsJzsV8aQ3Sdhr7SfUyp1cmYbwr8cN2uGpJUR0aOZ8oXAyIhYkCaSez0iekl6k2xKjPdT+aKI6C1p\nCVAfEatL9tEXmBURA9LzM4FuEXGBpHuBlcBdwF0RsbLCp2qWm1scZvlFC8t5rC5Z/pCPxhtHk80b\nNAz4Q5p11ayqOHGY5Teu5N/H0vLvyGZpBjgBeCQtPwh8HUBSV0k9W9qppC7AjhExBzgT6Al8rNVj\nVjR/mzFr3maSni55fm9ENF6Su7WkP5K1Go5LZd8Apkr6N2AJMCmVfxO4RtJJZC2Lr5PNrtqcrsAv\nUnIRcGlELG+3MzJrJx7jMMshjXEMj4g3i47FrCjuqjIzs1zc4jAzs1zc4jAzs1ycOMzMLBcnDjMz\ny8WJw8zMcnHiMDOzXP4LNw9blrDo4TIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb6b06deda0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting gradients\n",
    "n_layers = len(tf.trainable_variables()) // 2\n",
    "x = np.arange(epochs)\n",
    "i = 0\n",
    "plt.figure()\n",
    "while i < n_layers*2:\n",
    "    plt.errorbar(x,mean_gradients[i,:],std_gradients[i,:])\n",
    "    i = i + 2\n",
    "plt.yscale('log')\n",
    "plt.ylabel('Gradient')\n",
    "plt.xlabel('Epochs')\n",
    "plt.title('Weights Gradient by Layer')\n",
    "plt.legend([\"conv1\",\"conv2\",\"fc1\",\"fc2\"][-n_layers:])\n",
    "plt.xlim(-0.2, epochs)\n",
    "plt.show()\n",
    "i = 1\n",
    "plt.figure()\n",
    "while i < n_layers*2:\n",
    "    plt.errorbar(x,mean_gradients[i,:],std_gradients[i,:])\n",
    "    i = i + 2\n",
    "plt.yscale('log')\n",
    "plt.ylabel('Gradient')\n",
    "plt.xlabel('Epochs')\n",
    "plt.title('Biases Gradient by Layer')\n",
    "plt.legend([\"conv1\",\"conv2\",\"fc1\",\"fc2\"][-n_layers:])\n",
    "plt.xlim(-0.2, epochs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'CIFAR10' object has no attribute 'training_data'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-58da638e24cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m accs = sess.run(accuracy,\n\u001b[1;32m     26\u001b[0m              feed_dict={\n\u001b[0;32m---> 27\u001b[0;31m             \u001b[0mmodel_input\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcifar10\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m                        \u001b[0mtarget\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcifar10\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_labels\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m             \u001b[0mkeep_prob\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'CIFAR10' object has no attribute 'training_data'"
     ]
    }
   ],
   "source": [
    "def confusion_matrix(targets, outputs):\n",
    "    '''Returns a confusion matrix. Both targets and outputs\n",
    "    should be 1-D arrays of zeros and ones.'''\n",
    "    encoded_data = 2*targets+outputs  # Map targets and outputs to {0, 1, 2, 3}\n",
    "    TN = np.sum(encoded_data == 0)  # True negatives\n",
    "    FP = np.sum(encoded_data == 1)  # False positives\n",
    "    FN = np.sum(encoded_data == 2)  # False negatives\n",
    "    TP = np.sum(encoded_data == 3)  # True positives\n",
    "    return ((TP, FP), (FN, TN))\n",
    "\n",
    "def roc_curve(targets, outputs):\n",
    "    '''Returns a ROC curve. Outputs should be in range 0-1\n",
    "    in order to move the threshold.'''\n",
    "    tpr = []\n",
    "    fpr = []\n",
    "    for threshold in np.linspace(0, 1, 1000):\n",
    "        outputs_with_threshold = (outputs > threshold).astype(np.float)\n",
    "        ((TP, FP), (FN, TN)) = confusion_matrix(\n",
    "            targets, \n",
    "            outputs_with_threshold)\n",
    "        tpr.append(TP/(TP+FN))\n",
    "        fpr.append(FP/(FP+TN))\n",
    "    return np.array(tpr), np.array(fpr)\n",
    "\n",
    "accs = sess.run(accuracy,\n",
    "             feed_dict={\n",
    "            model_input: cifar10.training_data,\n",
    "                       target: cifar10.training_labels,\n",
    "            keep_prob: 1.0\n",
    "        })\n",
    "\n",
    "accs = np.asarray(accs)\n",
    "[[VN,FP],[FN,VP]] = confusion_matrix(\n",
    "    accs,\n",
    "    (accs>0.5).astype(np.float))\n",
    "print('VP: %d, VN: %d, FP: %d, FN: %d' %(VP,VN,FP,FN))\n",
    "print('Porcentaje de clasificaciones correctas: %%%f' %(100.0*(VP+VN)/(VP+VN+FP+FN)))\n",
    "print('Precision: %%%f' %(100.0*VP/(VP+FP)))\n",
    "print('Recall: %%%f' %(100.0*VP/(VP+FN)))\n",
    "print('')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}