Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.random.set_seed(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SPECTROGRAM_TRAIN_TFRECORD_FILENAME = \"spectrogram_train.tfrecord\"\n",
    "SPECTROGRAM_VALID_TFRECORD_FILENAME = \"spectrogram_valid.tfrecord\"\n",
    "SPECTROGRAM_TEST_TFRECORD_FILENAME = \"spectrogram_test.tfrecord\"\n",
    "DATA_DIR = \"data\"\n",
    "BUFFER_SIZE = 64\n",
    "BATCH_SIZE = 64\n",
    "SPECTROGRAM_SHAPE = (85, 129)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trained using a GPU cluster from the University of Edinburgh."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed May 12 11:06:09 2021       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 450.80.02    Driver Version: 450.80.02    CUDA Version: 11.0     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|                               |                      |               MIG M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  GeForce GTX 106...  Off  | 00000000:04:00.0 Off |                  N/A |\r\n",
      "| 32%   30C    P8     6W / 120W |      0MiB /  6078MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   1  GeForce GTX 106...  Off  | 00000000:05:00.0 Off |                  N/A |\r\n",
      "| 30%   35C    P8     6W / 120W |      0MiB /  6078MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   2  GeForce GTX 106...  Off  | 00000000:08:00.0 Off |                  N/A |\r\n",
      "| 30%   27C    P8     5W / 120W |      0MiB /  6078MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   3  GeForce GTX 106...  Off  | 00000000:09:00.0 Off |                  N/A |\r\n",
      "| 30%   31C    P8     6W / 120W |      0MiB /  6078MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   4  GeForce GTX 106...  Off  | 00000000:83:00.0 Off |                  N/A |\r\n",
      "| 31%   30C    P8     6W / 120W |      0MiB /  6078MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                                  |\r\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\r\n",
      "|        ID   ID                                                   Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|  No running processes found                                                 |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'),\n",
       " PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'),\n",
       " PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'),\n",
       " PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'),\n",
       " PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.config.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = tf.data.TFRecordDataset([os.path.join(DATA_DIR, SPECTROGRAM_TRAIN_TFRECORD_FILENAME)])\n",
    "valid_dataset = tf.data.TFRecordDataset([os.path.join(DATA_DIR, SPECTROGRAM_VALID_TFRECORD_FILENAME)])\n",
    "test_dataset = tf.data.TFRecordDataset([os.path.join(DATA_DIR, SPECTROGRAM_TEST_TFRECORD_FILENAME)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_description = {\n",
    "    'spectrogram': tf.io.FixedLenFeature([], tf.string, default_value=''),\n",
    "    'label': tf.io.FixedLenFeature([], tf.float32, default_value=0.0)\n",
    "}\n",
    "\n",
    "def _parse_function(example_proto):\n",
    "    example = tf.io.parse_single_example(example_proto, feature_description)\n",
    "    spectrogram = tf.io.parse_tensor(example['spectrogram'], out_type=tf.float32)\n",
    "    spectrogram = tf.ensure_shape(spectrogram, SPECTROGRAM_SHAPE)\n",
    "    label = tf.ensure_shape(example['label'], ())\n",
    "    return spectrogram, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:AutoGraph could not transform <function _parse_function at 0x000001F95FE603A0> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: module 'gast' has no attribute 'Index'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n",
      "WARNING: AutoGraph could not transform <function _parse_function at 0x000001F95FE603A0> and will run it as-is.\n",
      "Please report this to the TensorFlow team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output.\n",
      "Cause: module 'gast' has no attribute 'Index'\n",
      "To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n"
     ]
    }
   ],
   "source": [
    "train_dataset = train_dataset.map(_parse_function)\n",
    "valid_dataset = valid_dataset.map(_parse_function)\n",
    "test_dataset = test_dataset.map(_parse_function)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(<tf.Tensor: shape=(85, 129), dtype=float32, numpy=\n",
      "array([[5.50424635e-01, 3.23084682e-01, 7.39609897e-01, ...,\n",
      "        8.44748065e-05, 1.00848323e-04, 7.20620155e-05],\n",
      "       [8.08760464e-01, 5.43129921e-01, 9.12065446e-01, ...,\n",
      "        4.87966863e-05, 6.39859863e-05, 1.21831894e-04],\n",
      "       [4.52644348e-01, 1.53675258e+00, 5.54100394e-01, ...,\n",
      "        2.00303148e-05, 2.59448389e-05, 7.03334808e-05],\n",
      "       ...,\n",
      "       [6.99571013e-01, 8.03465366e-01, 7.09879398e-01, ...,\n",
      "        7.33799025e-05, 7.61434203e-05, 9.62615013e-06],\n",
      "       [3.28246504e-03, 1.66637897e+00, 2.48969793e+00, ...,\n",
      "        1.12396221e-04, 1.19734090e-04, 1.23858452e-04],\n",
      "       [1.18247461e+00, 2.05399251e+00, 1.59880507e+00, ...,\n",
      "        8.77474740e-05, 2.21014307e-05, 2.74181366e-05]], dtype=float32)>, <tf.Tensor: shape=(), dtype=float32, numpy=0.0>)\n"
     ]
    }
   ],
   "source": [
    "for i in train_dataset.take(1):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.batch(BATCH_SIZE).prefetch(BUFFER_SIZE).cache()\n",
    "valid_dataset = valid_dataset.batch(BATCH_SIZE).prefetch(BUFFER_SIZE).cache()\n",
    "test_dataset = test_dataset.batch(BATCH_SIZE).prefetch(BUFFER_SIZE).cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# First model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model parameters copied from tutorial: https://www.tensorflow.org/tutorials/audio/simple_audio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "reshape (Reshape)            (None, 85, 129, 1)        0         \n",
      "_________________________________________________________________\n",
      "resizing (Resizing)          (None, 32, 32, 1)         0         \n",
      "_________________________________________________________________\n",
      "normalization (Normalization (None, 32, 32, 129)       259       \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 30, 30, 32)        37184     \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 28, 28, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 12544)             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               1605760   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 1,661,957\n",
      "Trainable params: 1,661,698\n",
      "Non-trainable params: 259\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "norm_layer = tf.keras.layers.experimental.preprocessing.Normalization()\n",
    "norm_layer.adapt(train_dataset.map(lambda x, _: x))\n",
    "model1 = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=SPECTROGRAM_SHAPE),\n",
    "    tf.keras.layers.Reshape((SPECTROGRAM_SHAPE[0], SPECTROGRAM_SHAPE[1], 1)),\n",
    "    tf.keras.layers.experimental.preprocessing.Resizing(32, 32),\n",
    "    norm_layer,\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(128, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.5),\n",
    "    tf.keras.layers.Dense(2)\n",
    "])\n",
    "model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
    "               metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "163/163 [==============================] - 21s 90ms/step - loss: 471.9781 - accuracy: 0.8021 - val_loss: 0.3514 - val_accuracy: 0.8637\n",
      "Epoch 2/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.3855 - accuracy: 0.8871 - val_loss: 0.2418 - val_accuracy: 0.9426\n",
      "Epoch 3/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.2804 - accuracy: 0.9211 - val_loss: 0.2100 - val_accuracy: 0.9538\n",
      "Epoch 4/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.2495 - accuracy: 0.9269 - val_loss: 0.2237 - val_accuracy: 0.9484\n",
      "Epoch 5/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.2884 - accuracy: 0.9250 - val_loss: 0.1836 - val_accuracy: 0.9555\n",
      "Epoch 6/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.2207 - accuracy: 0.9331 - val_loss: 0.1710 - val_accuracy: 0.9582\n",
      "Epoch 7/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.2029 - accuracy: 0.9411 - val_loss: 0.1671 - val_accuracy: 0.9585\n",
      "Epoch 8/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.1950 - accuracy: 0.9429 - val_loss: 0.1826 - val_accuracy: 0.9578\n",
      "Epoch 9/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.1963 - accuracy: 0.9445 - val_loss: 0.1309 - val_accuracy: 0.9663\n",
      "Epoch 10/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.1717 - accuracy: 0.9486 - val_loss: 0.1354 - val_accuracy: 0.9690\n",
      "Epoch 11/20\n",
      "163/163 [==============================] - 2s 12ms/step - loss: 0.1919 - accuracy: 0.9438 - val_loss: 0.1776 - val_accuracy: 0.9565\n"
     ]
    }
   ],
   "source": [
    "history1 = model1.fit(train_dataset, epochs=20, validation_data=valid_dataset, callbacks=tf.keras.callbacks.EarlyStopping(patience=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Primary analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the base model already does pretty well, getting a validation accuracy of over 90%."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAx80lEQVR4nO3deXzU1b3/8dcnG1nInpAdEghbCIuILIKK4gICYtV6sbX+yq+t1Wqr3tuq11tr7Wp7r73VX7Vcamlvr1SvRa1AAbeqVAmrkJAQ1oQsZCEhZCEkZDu/P74DDCGQSZgtk8/z8chjMjPf78xnXN5zcs75niPGGJRSSvkuP08XoJRSyrU06JVSysdp0CullI/ToFdKKR+nQa+UUj4uwNMF9CQuLs6kp6d7ugyllBowdu7cWWuMie/pOa8M+vT0dHbs2OHpMpRSasAQkZKLPaddN0op5eM06JVSysdp0CullI/zyj76nrS3t1NeXk5ra6unS3G54OBgUlNTCQwM9HQpSikfMGCCvry8nPDwcNLT0xERT5fjMsYYjh8/Tnl5ORkZGZ4uRynlAwZM101rayuxsbE+HfIAIkJsbOyg+MtFKeUeAyboAZ8P+TMGy+dUSrnHgOm6UUopl2ltgLoiOH4YGo/C6Jth2HhPV+U0GvQOqq+v589//jPf+ta3+nTerbfeyp///GeioqJcU5hSyjGtjVaY1x2G40V2vx+GU7XnH/v3n8B1j8PsR8F/4E+K0KB3UH19PS+//PIFQd/Z2Ym/v/9Fz1u/fr2rS1NKnXG66VzLvO4w1BWf+7255vxjw5MgZhSMuxViRlq/x46CIRHw/tNW2BeuhSUvQ2K2Zz6Pk2jQO+jJJ5/k8OHDTJkyhcDAQIYOHUpSUhK7d+9m79693H777ZSVldHa2sojjzzC/fffD5xbzuHkyZMsWLCAOXPmsHnzZlJSUnjnnXcICQnx8CdTaoA5ffJca7yuyNY6t7XMm4+df+zQRCu8x8y3bs8EekwGBIVd/D2++EfIuh3+9i+wYq7Vup/z2IBt3Q/IoH92bQF7Kxqd+ppZyRE8s3jCRZ9/7rnnyM/PZ/fu3Xz88ccsXLiQ/Pz8s1MgV65cSUxMDC0tLVx11VXceeedxMbGnvcaBw8e5LXXXuN3v/sdd999N2+++Sb33nuvUz+HUj6hqxOOFcLxQxd2t5ysOv/YoQlWeI+52RbiI8+F+qXCvDcTbof0a2DD9+Cjn0LhGqt1nzTpsj6aJwzIoPcG06dPP2+e+4svvsjbb78NQFlZGQcPHrwg6DMyMpgyZQoAV155JUeOHHFXuUoNDG3NsPvPsOVlK9TPGJpgBXfmjRA70q5lPhKGDHVdPWGxcNdKmPAFWPfP8Lvr4ZrvwjX/AgFBrntfJxuQQX+plre7hIWdayl8/PHHfPDBB+Tk5BAaGsrcuXN7nAc/ZMiQs7/7+/vT0tLillqV8npNVbBtBexYCS0nIGUaLPmu1TceMxKGhHu2vvGLYcRs2PAEfPIc7FsHt78MSZM9W5eDBmTQe0J4eDhNTU09PtfQ0EB0dDShoaHs27ePLVu2uLk6pQaoqnzIeQn2/AW6OmD8Ipj1MKTNAG+7niQ0Bu78ndWls+4x+N0NMOef4drveX3rXoPeQbGxscyePZvs7GxCQkJISEg4+9z8+fNZvnw5kyZNYuzYscycOdODlSrl5YyBwx/C5t9A0UcQGArTlsHMB63Wu7cbtxCGz4J3n4JNv4R9f4PbX4LkKzxd2UWJMcbTNVxg2rRppvvGI4WFhYwf7zsXMPRmsH1eNQh0nIa8N6wWfE2hNSNmxjfhyq9areWBaP9GWPconDwGcx6F656AgCG9neUSIrLTGDOtp+e0Ra+Ucq3m47Dj97Dtd9b0x4RsuH05ZN/p9V0evRo7H4bnwLv/Bv94Hvatt1r3KVd6urLzaNArpVyj9hBseQl2vwYdLZB5E1z9MGRc533975cjJNoamJ3wBVjzHXjlRpj9CFz3JAQGe7o6QINeKeVMxkDJZ1b/+4GN4B8Ek/8JZj4Ew8Z5ujrXGn0TPLTF6rv/9D9trfuXIbXH3hS3cmj1ShGZLyL7ReSQiDzZw/PRIvK2iOSJyDYRybZ7LkpEVovIPhEpFJFZzvwASikv0NkOe1ZbV5H+cSGUb7OuJn0sH277f74f8mcER8KSl+DLb0LbSfj9TfDe09Du2anUvbboRcQfeAm4CSgHtovIGmPMXrvDngJ2G2O+ICLjbMfPsz33ArDRGHOXiAQBoU79BEopz2ltgJ3/DVv/CxrLIXY0LPpPmHwPBA7i5T1G3wjfyrFCfvOLsH+D1bpPm+6RchzpupkOHDLGFAGIyOvAEsA+6LOAnwMYY/aJSLqIJAAtwLXAV23PtQFtTqteKeUZJ0pg63L4/E9WyzX9Glj4vLW8r9+A2ubCdYIj4bYXIWsJrH0Efn8zzHoIbvi+278EHfk3kgKU2d0vtz1mLxe4A0BEpgMjgFRgJFAD/EFEdonIKyLS4+ITInK/iOwQkR01NTU9HTKgDB1qXZZdUVHBXXfd1eMxc+fOpfs0UqW8WvlO+MtX4cUp1pWsY2+F+z+Br66zZqBoyF8ocx48uNm6ViDnN7B8DpS696JKR1r0PQ2Pd598/xzwgojsBvYAu4AOIBCYCnzbGLNVRF4AngSevuAFjVkBrABrHr2jH8DbJScns3r1ak+XoZyhq9NqvXZ2WFdxdrVbt532v7dbx513v+PC3/v6nOmCgGBrFkdAiNUiPPu73W1gqO24kHO3gSHW8/79nHvR1Qn711vz30tzYEikdfXqjAcgsnubT/UoOMLq0spaAu98G1bOty4Qu+FpCHJ9b7Yj/+bLgTS7+6lAhf0BxphGYBmAWPvgFdt+QoFyY8xW26GrsYJ+wHniiScYMWLE2fXof/jDHyIibNq0iRMnTtDe3s5PfvITlixZct55R44cYdGiReTn59PS0sKyZcvYu3cv48eP17VuvFFXJzSU2dYwLzp/bfMTJVaAu4NfAPgFWrf+ASB+1gVH7S1gOvv/mud9MfT0ZRFy/peEn7+1JntdEUQNh/nPwRX3en7tmYFq5Fz41mb44IfWwm0HNlqDtyOudunbOhL024HRIpIBHAWWAl+yP0BEooBTtj74rwObbOHfKCJlIjLWGLMfa4B2L5drw5NQteeyX+Y8iRNhwXMXfXrp0qU8+uijZ4P+jTfeYOPGjTz22GNERERQW1vLzJkzue222y665+tvf/tbQkNDycvLIy8vj6lTpzr3MyjHdHVCQ3nP65mfOHJ+mAeGWpflD8uCcYsgLN5ak9zP3wpif1sYn/nxD7QFtL/dc/b3e3suwLp/qXnmne1W4Le3WPPT21vP3bafgo5W23Pdbi/6WAu0nYJTx22vZXudM7+nTIV5P4Bxi/v/V4E6Z0i4NZ6RtQTeeRj+cKt1hfC8H1zessqX0Ou/NWNMh4g8DLwL+AMrjTEFIvKA7fnlwHjgTyLSiRXkX7N7iW8Dq2wzboqwtfwHmiuuuIJjx45RUVFBTU0N0dHRJCUl8dhjj7Fp0yb8/Pw4evQo1dXVJCYm9vgamzZt4jvf+Q4AkyZNYtKkgbeu9YDR1WXNAjnbIre7PVEMnXZzAgJCbGE+zrbbkG2noZhREJ7ofRf3+Nu+YIIjPF2JuhwZ11p99x/+yBrYPtO6T5/j9Ldy6OvZGLMeWN/tseV2v+cAoy9y7m7AuVcMXKLl7Up33XUXq1evpqqqiqVLl7Jq1SpqamrYuXMngYGBpKen97g8sb2LtfZVP3R1WRs52+/9eaa7pa4YOk+fOzYg2ArzuNEw5pZzQR47ylpzRQcRlScMGQq3/hKybrNa92/cB4/ucXrLXv8O64OlS5fyjW98g9raWj755BPeeOMNhg0bRmBgIB999BElJSWXPP/aa69l1apVXH/99eTn55OXl+emyn1Ee4u1UmDhGqg5YLXMO+y+WP2H2HYXyrSuUjyvZZ6kYa68V/ocePAzqNnvku4bDfo+mDBhAk1NTaSkpJCUlMSXv/xlFi9ezLRp05gyZQrjxl366r8HH3yQZcuWMWnSJKZMmcL06Z65eGJAMQaO7oRdr0L+W3C6AcKTrSVhM+edC/KYkRCRomGuBq6gMGs8xAV0mWIvNdg+7wUaKyHvdWtbudoDVj961m0w5UuQfq0GulLd6DLFvqirC3a/ai2eFJFidVVk3gTDxnvf4KGj2lvhwAbYtcramMJ0QdpMWPyitTKgDj4q1S8a9ANR+U5Y/12o+BySp1p7bL7/A+snItVaZ2P0zdZysK7cONkZjIGKXbB7lbUoVmu99cU15zGY/CWIy/R0hUoNeAMq6I0xg2LWykW7007WwIc/tPqrhybAF1bApLutFnzDUTj0ARx8D/a8CTv/aM3LHjHLCv3MmyB+rPe09puqIe9/ra6ZmkJrVsy4RVbXzMi51lxypZRTDJg++uLiYsLDw4mNjfXpsDfGcPz4cZqamsjIyLAe7OyA7a/ARz+D9mbr0ulrH794V0ZHG5RtgYPvWz81hdbjkcOt1n7mTdYcXne39jvarLnCu1dZdZlOSL3KCvcJd0BIlHvrUcqHXKqPfsAEfXt7O+Xl5b3OU/cFwcHBpKamEhgYCEc+hfWPw7ECGHk9LPglxI/p2wvWl9la++9D0cfWl4V/kHXZ9ZnWftxo17T2jYGqPKvffc9foKXOmrc+eakV8PFjnf+eSg1CPhH0g07DUXjv+1DwltUSn/8zq2vjcsO447S1MNWZ1n7tfuvxqBHnBnQzrrn8ubwna6xg370KqvOtL5ZxC2HKl60vLL2UXimn0qAfSDpOW6sEbvp3a9bJ7Eet/SddtcLdiZJzrf3iT6w1TvyHQPrsc6392FGOfcF0tltjBLtWwcF3rZUXk6daLffsOyE0xjWfQSmlQT9gHHgPNj5pXc4/bhHc8lOITnff+3ectvb7PGgb1D1+0Ho8Ov1c6KfPufBLpyrfarnnvQGnaiFsmLVP6OQvQUKW++pX6jIZY2hp7yQ0aOD9xalB7+3qimDjU9Yc8thMWPALyLzR01VZKzme6eIp3mStchgQbIV95k3WMbl/hspca4bP2AVW10zmjdo1o7zSydMdVNa3UNHQet5tVWMrFfUtVDa0cqqtk9mZsfxoSTaj4r18erIdDXpv1XYKPv0VfPaitRrhdY/DjAchIMjTlV2ovRVKPj3X2q87bD2eNNkK94lf1K4Z5VEtbZ1UNLRQWd9KZYMV2pUNLVScuV/fStPpjvPOEYH4oUNIigohOTKYpMgQQoL8+FNOCa3tnXzz2lE8dH0mIUHeP91Xg97bGAN734F3/81aSnfi3XDTjyAiydOVOa6uyOqT11kzyg1Od3RS1dB6LrQbzoV3he33+lMXbgoTNzSIRFuAJ0cGkxQVQlJkMMm222HhwQQFXLicRk3TaX6+oZC3Pj9KanQIz942gXnjE9zxUftNg96bHNsHG75ndYUkTLSWKHXx7jJKebOuLkPtydMcrbda3xX1LRy1/VTZQrz2ZNsF50WFBpIUaQW2fXgnRYaQHBVMQkQwwYGX1xLfUnScp/+az8FjJ7kpK4FnFmeRGu36rf/6Q4PeG7Q2wMe/gG3/ZU1dvOFpuHKZ9mUrn9fS1mkL8XM/R+0CvbKhhfbO83MoLMif5KgQ208wiREhJEUFkxxp3SZFBrttwLS9s4uVnxbz6w8OYjB8Z95ovj5nZI9/CXiSBr0ndXVZqzC+/ww018DU+6wtw8LiPF2ZUpetq8tQ23z6XEv8RMu5ULf1j9c1n98a9xNIiAg+G+QpUSGkRAXbBXsIEcEBXncF/NH6Fn68di8bC6oYFR/Gj2/P5upR3vP/sQa9p1TshvXfg/Jt1qX+C37psvWmlXKF1vbzW+NH7QL9zMBnW2fXeeeEBfmTEh3SLcjPtc4TIoIJ9Peu1nBffLTvGM+sKaC07hS3T0nmqYXjGRYe7OmyNOjdrvk4/P1HsPO/rZb7jc/C5Ht0DXXl9ZpPd7Cz5ARbi4+zrbiO3LKG84K8e2s8OSrYCvHIkLPh7o2tcWdrbe/k5Y8Ps/zjwwwJ8OO7t4zl3pkj8Pfz3OfWoHeXrk7YsRL+/hM43WTt7D73SQiO9HRlSvWoqbWdHUdOsKX4OFuL6sg/2kBHl8HfT8hOiWRmRgzjksJJjrRCPDFyYLfGna24tpkfvJPPPw7Wkp0SwU9un8iUtCiP1KJB7w4lOdZsmqo9kH4N3Prv1iYgSnmRhlPtbDtSx9ai42wtrqOgooEuA4H+wqTUKGZkxDBjZCxXjohm6BCdKOAIYwzr91Txo3UFHGs6zT3Th/P4LWOJCnXv9TAa9K6W+zq8/U1rw4xbfgpZt3vPuu9qUKtrbmNb8XG2FNWxtbiOfVWNGANBAX5MSYtipi3Ypw6PHhAXBXmzptZ2fv3BQf64+QiRIYH864Jx3HVlqtu6sTToXe3Vu6wrRR/41CU7uCvlqGNNrWwrrmNrUR1bi49zoPokAMGBflw5IpoZGbHMyIhhclrUZc8xVz3bW9HI0+/ks7PkBFelR/Pj27MZl+j6bTAve89YEZkPvAD4A68YY57r9nw0sBIYBbQC/9cYk2/3vD+wAzhqjFnUr0/hrbo6oWwrZN+hIa/crqqhla1nW+zHKappBqyZL1emx7BkSgozR8YwMSXK6+Z9+6qs5Aj+8s1ZrN5Zzs83FLLwxU/52pwMHpk3mjAPdYf1+q62kH4JuAkoB7aLyBpjzF67w54CdhtjviAi42zHz7N7/hGgEPC93Z2rC+B0IwzXq1uV65WfOHW2tb61uI6S46cACB8SwFUZMfzTtDRmjIwlOzmCAB009Rg/P+Huq9K4KSuBX767jxWbilizu4JnFmcxPzvR7bOSHPl6mQ4cMsYUAYjI68ASwD7os4CfAxhj9olIuogkGGOqRSQVWAj8FPhnp1bvDUpzrNsRszxbh/I5dc1t7KtqZF9lE/lHG9haXMfR+hbAuvx/enoM981KZ0ZGDOOTIjw6tU/1LDosiJ/fMYkvTkvj+2/n8+Cqz7luTDzP3jaB9Dj39QA4EvQpQJnd/XJgRrdjcoE7gE9FZDowAkgFqoFfA48D4Zd6ExG5H7gfYPjw4Q6U5SVKNkNEKkQNoJqVV2nr6OJwzcmzoV5Y1cT+qkaqG0+fPSZu6BCmZ0Rz/7UjmTEyhjHDwvHTYB8wpg6PZs3Ds/mfLSU8/94Bbv71Jh68bhQPzh3llrESR4K+p/+auo/gPge8ICK7gT3ALqBDRBYBx4wxO0Vk7qXexBizAlgB1mCsA3V5njFWiz7jWk9XogYAYwzVjacprGpkf1UT+yob2VfVxKFjJ+nosv6TD/L3Y3TCUOZkxjM+KZxxiRGMTQwnPnyIh6tXlyvA349lszNYODGJn/ytkBc+PMhfdx/l2dsmMHfsMNe+twPHlANpdvdTgQr7A4wxjcAyALE6n4ptP0uB20TkViAYiBCRV40x9zqhds+rK4KT1TBcu23U+VraOjlQ3cS+qkYKK63bfVVN5y2lmxIVwtjEcG4YN4xxSRGMTwwnPS5ML0jyccMignnxniv4p6vSePqdfL76h+0syE7k6UVZJEeFuOQ9HQn67cBoEckAjmKF95fsDxCRKOCUMaYN+DqwyRb+/2r7wdai/67PhDzY9c/rQOxg1dVlKD/RQqGt2+VMoB853syZmcuhQf6MTQxnQXbSuVZ6QjiRoYGeLV551OzMODY8cg2v/KOYFz88yCcHanj0xtEsm53h9C/7XoPeGNMhIg8D72JNr1xpjCkQkQdszy8HxgN/EpFOrEHarzm1Sm9VkgMh0RCnm28MBi1tnew52nA2zPdVWl0wzW2dgHWNXHpsGOMSw7l9SgpjE8MZnxROWnSo9qerHg0J8Oeh6zO5bXIyP1xTwP9uL+OrV2c4/X30gqnL8eIVED8O7nnN05UoF3u3oIrv/zWfmiZrgDQyJJBxieGMT4pgXGI445IiGJMwdEBuKq28R/2ptn4vnXDZF0ypHjRVW330Vy7zdCXKhY6fPM0zawpYl1fJ+KQIfvaFiWSnRJAYEezzKzQq93PV+jga9P1Vutm61f55n2SMYW1eJT9cU8DJ1g6+e/MYvnndKB0oVQOSBn1/leRAYCgkTfZ0JcrJqhtb+f5f83l/bzWT06L497smMSbhkpeBKOXVNOj7q3QzpE4Df5054SuMMazeWc6P1+3ldEcX/3breP7vnAy94lQNeBr0/dHaAFX5cN0Tnq5EOUn5iVM89XY+mw7UMD0jhl/cOYkMN16irpQradD3R9k2wOj6Nj6gq8uwalspz60vxAA/XjKBL88YodMhlU/RoO+Pks3gF2Bt+K0GrCO1zTzxZh5bi+u4ZnQcP/vCRNJiQj1dllJOp0HfH6U51iCsrj8/IHV2Gf7wWTH/8d5+Av39+OWdk/jiNPftBKSUu2nQ91V7KxzdCdPv93Qlqh8OVjfx+Jt57Cqt58bxw/jpFyaSEBHs6bKUcikN+r6q+Bw62wbl/PkTzW3kFB3n00O1fHaoltPtXdyUlcD87ERmZMR49UYX7Z1drNhUxAsfHCRsiD8vLJ3CbZOTtRWvBgUN+r4qsV0oNQhWrGxt72THkRNngz2/ogFjrN2MZo6KJcBPWL2znP/ZUkJ0aODZ0J+dGceQAO/Zj7SgooHHV+dRUNHIwklJPHvbBOKG6rK/avDQoO+r0hxrfZvQGE9X4nSdXYa9FY1ng337kTpOd3QR6C9cMTyax24cw+zMOCanRp5tvbe0dfLJgRreLahiQ34Vb+woJ3xIADeMH8b8CYlcNzbeY+u/nO7o5KW/H+Lljw8TFRrE8nunMj87ySO1KOVJGvR90dVpTa3MvtPTlThN6fFTfHqolk8P1bD58PGz66WPSwzn3pkjmDM6junpMRfd1DgkyJ/52YnMz06kraOLzYdr2ZhfxXt7q3lndwXBgX5cNyaeBdlJ3DB+GBHB7rnAbFfpCR5fncfBYye5Y2oKP1iU5bJ1RJTydhr0fVGdb20EPmK2pyvpt7rmNjYftlrsnx6qpazO2oM0KTKYm8YnMGd0HLNGxTIsvO8DlEEBfswdO4y5Y4fxk9u72H7kBBvzK9lYUMW7BdUE+guzM+NYkJ3IjeMTiHVB90lreye/ev8Ar/yjiISIYP6w7Cqud/HuPUp5Ow36vigZeBuBt7Z3sv1I3dnumIKKxrP97LNGxfKNa0YyOzOOkXFhTh2YDPD3Y9aoWGaNiuWZxRPYXV7PxvwqNuRX8sSbe/CTPczIiGXBxERuzkokMfLyZ75sK67jiTfzKK5t5kszhvOvC8YR7qa/IJTyZroefV+8cR8c3QWP7fF0JRfV2WUoqGjgHwetYN9RcoI2Wz/71OHRzMmMY/boOCalRHpklowxhr2VjWzMr2JjfhUHj50EYOrwKOZnJ7IgO6nPFy01n+7gFxv38aecEtJiQvjFHZO4OjPOFeUr5bUutR69Br2jjIH/GAOjroc7Vni6mrOMMZTY+tk/O1TL5sPHaWg5188+JzPO6mfPiPHKTTEOHTtpG8itJP9oIwATkiOYPyGRBRMTyRx26VUj/3Gwhiff3ENFQwtfvTqd790y1is/p1KuphuPOENdETQf86pplVUNrdy3cisHqq1WcXJkMLdMSGB2ZhxXj4ojPtz7pxBmDhtK5rBMHro+k7K6U2dn7zz//gGef/8Ao+LDWJCdxPzsRCYkR5ztXmpoaednfyvkf3eUMTI+jNUPzOLKEb43E0opZ9Cgd1TJZ9atl1wodbqjkwde3Un5iRaevW0C14yOI8PJ/ezulhYTytevGcnXrxlJdWMr79lC/7efHOY3Hx0iLSaE+RMSyRw2lF+9f4Dak208OHcUj8wbTXCg98zbV8rbaNA7qiQHQmMhboynK8EYwzPvFLC7rN5n54YnRATzlVnpfGVWOnXNbXywt5oN+ZX8cfMR2jsN4xLD+d1905iUGuXpUpXyehr0jirdbHXbeEGLedXWUl7fXsbD12f6ZMh3FxMWxN1XpXH3VWk0trZTWNHIFcOjCQrw3iUXlPIm+n+KIxor4cQRr+if33GkjmfXFnD92Hgeu8nzf124W0RwIDNGxmrIK9UH+n+LI85uBO7ZoK9qaOWBVz8nJSqEXy+9Qre4U0o5xKGgF5H5IrJfRA6JyJM9PB8tIm+LSJ6IbBORbNvjaSLykYgUikiBiDzi7A/gFiU5EBgGiZ7bCPzM4GtLWwcr7ptGZIheCKSUckyvQS8i/sBLwAIgC7hHRLK6HfYUsNsYMwm4D3jB9ngH8C/GmPHATOChHs71fqU5kHYV+HtmSMMYww/+ag2+Pn/3ZMYkXHpuuVJK2XOkRT8dOGSMKTLGtAGvA0u6HZMFfAhgjNkHpItIgjGm0hjzue3xJqAQSHFa9e7QUg/VBTDcc9MqV20t5X93DJ7BV6WUczkS9ClAmd39ci4M61zgDgARmQ6MAFLtDxCRdOAKYGtPbyIi94vIDhHZUVNT41DxblG2FU9uBD7YB1+VUpfPkaDvacSv+7oJzwHRIrIb+DawC6vbxnoBkaHAm8CjxpjGnt7EGLPCGDPNGDMtPj7ekdrdo2Qz+AVCSo9XFrvUmcHX1OhQHXxVSvWbI53O5UCa3f1UoML+AFt4LwMQ69LMYtsPIhKIFfKrjDFvOaFm9yrNgeQpENS3hbYul/3g62vfmKGDr0qpfnOkRb8dGC0iGSISBCwF1tgfICJRtucAvg5sMsY02kL/90ChMeZXzizcLdpb4Ojnbp8/333wdbQOviqlLkOvLXpjTIeIPAy8C/gDK40xBSLygO355cB44E8i0gnsBb5mO3028BVgj61bB+ApY8x6534MFzm6E7ra3b6+zas6+KqUciKH5gvagnl9t8eW2/2eA4zu4bxP6bmPf2A4s9FI2gy3veX2I3U8u0YHX5VSzqNXxl5K6WYYluW2jcCrGlp58NXPSYvRwVellPNo0F9MZ4e1Ebib+ufPu/L1K1fq4KtSyml09cqLqd4DbSfd0j9vP/i6/N4rdfBVKeVU2qK/mDP9825o0Z8ZfP32DZnMz050+fsppQYXDfqLKd0MUcMh0rUrNtgPvj56ow6+KqWcT4O+J8ZYLXoXr29T2dCig69KKZfTPvqeHD8Ep2pdur5Na3snD7z6uV75qpRyOQ36npTYNhpxUYveGMMP3sknVwdflVJuoF03PSnNgdA4iLvgGjCneHVrKW/sKNfBV6WUW2jQ96RkMwyf6ZKNwM+78lUHX5VSbqBB311jBdSXuGT+fPfBVz8dfFVKuYH20Xd3tn/euQOxOviqlPIUDfruSnMgaCgkTnLaS+rgq1LKk7TrpruSHEh17kbgr24p0cFXpZTHaNDbazkBx/Y6tX9+W3Edz67dyw3jhungq1LKIzTo7ZXaNgJ3Uv98ZUML31q1k7SYUP7zn6bo4KtSyiM06O2V2jYCT738jcDPDb526rLDSimP0sFYeyU5kDIVAkMu62V08FUp5U20RX9GewtU7HJKt40OviqlvIkG/RnlO5yyEbgOviqlvI0G/RmlOYBc1kbgOviqlPJGGvRnlGyGhAkQEtWv01vbO3ngf3bq4KtSyuto0IO1EXj59n73zxtjePqv+eSWN/D83VN08FUp5VU06AGq8mwbgfcv6N/edZS/7NTBV6WUd3Io6EVkvojsF5FDIvJkD89Hi8jbIpInIttEJNvRc71C6ZmNwPs3EPvqlhLGJAzVwVellFfqNehFxB94CVgAZAH3iEhWt8OeAnYbYyYB9wEv9OFczyvZDNHpEJHU51PLT5zi89J6lkxJ0cFXpZRXcqRFPx04ZIwpMsa0Aa8DS7odkwV8CGCM2Qeki0iCg+d6ljFQuqXfrfm/5VUCsGhS378klFLKHRwJ+hSgzO5+ue0xe7nAHQAiMh0YAaQ6eC628+4XkR0isqOmpsax6p2h9uBlbQS+Lq+SSamRjIgNc3JhSinlHI4EfU/9Eabb/eeAaBHZDXwb2AV0OHiu9aAxK4wx04wx0+Lj4x0oy0lK+78R+JHaZvYcbWDxpGQnF6WUUs7jyFo35UCa3f1UoML+AGNMI7AMQEQEKLb9hPZ2rseV5EBYPMSO6vOp6/Ksj7JQu22UUl7MkRb9dmC0iGSISBCwFFhjf4CIRNmeA/g6sMkW/r2e63Glm6358/3YCHxtbiXTRkSTHHV5i6AppZQr9Rr0xpgO4GHgXaAQeMMYUyAiD4jIA7bDxgMFIrIPa4bNI5c61/kfo58ajkJ9ab/WtzlQ3cT+6iYWT9ZuG6WUd3NomWJjzHpgfbfHltv9ngOMdvRcr3F2/nzfB2LX5VbgJ7Bgol4gpZTyboP7ytiSzyAoHBIn9uk0Ywzr8iqZOTKWYeHBLipOKaWcY5AHfQ6kTQc//z6dVlDRSFFtM4t0to1SagAYvEF/qg5qCvs1f35dXiUBfqLr2iilBoTBG/SlW6zbPs6fN8awNreC2ZlxxIQF9X6CUkp52CAO+s3gHwQpV/bptF1l9Rytb9HZNkqpAWPwBn1JDiRPhcC+Daauy60kyN+PmyckuKgwpZRyrsEZ9G3NULm7z/3zXV2Gv+2p4Lqx8UQE6w5SSqmBYXAGffkO6Oroc//89iN1VDee1pUqlVIDyuAM+rMbgU/v02lr8yoIDvTjxvHabaOUGjgGZ9CXbIaE7D5tBN7R2cWGPVXMG5dA2BCHLihWSimvMPiCvrPd2gi8j/3zOUXHOd7cxuLJ2m2jlBpYBl/QV+ZB+6k+r2+zLreSsCB/5o4d5qLClFLKNQZf0J/ZaKQPK1a2dXSxIb+SmyckEhzYt+USlFLK0wZf0JfkQHQGhDu+fMGnh2pobO3Q2TZKqQFpcAV9V5c146aP68+vza0kIjiAa0a7cYtDpZRyksEV9LUHoKWuT/3zre2dvL+3mgXZSQQFDK5/XEop3zC4kqsf/fMf7z/GydMdLNLZNkqpAWpwBX1JDoQNg5iRDp+yNq+S2LAgZo2MdWFhSinlOoMr6EtzrPnzDm4E3ny6gw8Lq1kwMZEA/8H1j0op5TsGT3rVl0FDGYyY7fApHxRW09rexWLdSUopNYANnqDvx0bg6/IqSYgYwlXpMS4qSimlXG/wBH3JZhgSAQkTHDq8oaWdT/bXsHBiMn5+jnX1KKWUNxo8QV+aA2kzHN4I/P291bR1dulsG6XUgOdQ0IvIfBHZLyKHROTJHp6PFJG1IpIrIgUisszuucdsj+WLyGsi0rctnZzhVB3U7OvTQmZrcytIiQrhirQo19WllFJu0GvQi4g/8BKwAMgC7hGRrG6HPQTsNcZMBuYCz4tIkIikAN8BphljsgF/YKkT63fM2f55x+bP1zW38dmhWhZNTkIcnKGjlFLeypEW/XTgkDGmyBjTBrwOLOl2jAHCxUrFoUAd0GF7LgAIEZEAIBSocErlfVGyGfyHQMpUhw7fmF9FR5fR2TZKKZ/gSNCnAGV298ttj9n7DTAeK8T3AI8YY7qMMUeB/wBKgUqgwRjzXk9vIiL3i8gOEdlRU1PTx4/Ri9IcSLkSAoY4dPi6vAoy4sKYkBzh3DqUUsoDHAn6nvouTLf7twC7gWRgCvAbEYkQkWis1n+G7bkwEbm3pzcxxqwwxkwzxkyLj3fi4mFtzVCZ63D//LGmVrYUHWfxJO22UUr5BkeCvhxIs7ufyoXdL8uAt4zlEFAMjANuBIqNMTXGmHbgLaBvS0dervLtfdoIfMOeKroMLJqs3TZKKd/gSNBvB0aLSIaIBGENpq7pdkwpMA9ARBKAsUCR7fGZIhJq67+fBxQ6q3iHlOSA+Dm8Efja3ArGJoQzJiHcxYUppZR79LrLtTGmQ0QeBt7FmjWz0hhTICIP2J5fDvwY+KOI7MHq6nnCGFML1IrIauBzrMHZXcAK13yUiyi1bQQe3Ht/e0V9CztKTvDdm8e4oTCllHKPXoMewBizHljf7bHldr9XADdf5NxngGcuo8b+62yH8h0w9T6HDv9bXiUAi3S2jVLKh/j2lbGVuX3aCHxdXgUTUyJJjwtzcWFKKeU+vh30JY5vNFJyvJnc8gbdF1Yp5XN8O+hLcyBmFAwd1uuh62zdNgs16JVSPsZ3g/7sRuCOddusza1g6vAoUqNDXVyYUkq5l+8Gfe1+aDnh0Pz5Q8ea2FfVxGKdO6+U8kG+G/Rn++d7b9Gvza1EBG6dqN02Sinf47tBX5oDQxMhOuOShxljWJtXwYyMGBIi3L+CslJKuZrvBn2JYxuBF1Y2UVTTrHPnlVI+yzeDvr4UGssd6p9fm1eBv5+wIDvRDYUppZT7+WbQl9g2Gumlf94Yw7q8Cq4eFUvsUMeWMFZKqYHGN4O+dDMMiYRh3TfCOl9ueQNldS0620Yp5dN8M+hLcmB47xuBr8utINBfuCVLu22UUr7L94K++bg1h76X9W26ugzr8iq5bkw8kaGBbipOKaXcz/eC/sxG4L2sb7Oz9ARVja3abaOU8nm+GfT+QyD5iksetja3giEBfswbn+CmwpRSyjN8L+hLPoPUaZfcCLyjs4v1eyqZN34YQ4c4tCS/UkoNWL4V9KdPQmVer/3zW4vrqD3ZphdJKaUGBd8K+vJtYDp7nT+/Lq+CsCB/rh/b+/LFSik10PlW0J/ZCDz14huBt3V0sSG/ihuzEggJuvT0S6WU8gW+FfSlOZA48ZIbgX92qJb6U+0s1m4bpdQg4TtB39EG5dt7Xd9mbV4F4cEBXDMmzk2FKaWUZ/nOlBO/APjaexA09KKHtLZ38l5BNQuyExkSoN02SqnBwYeC3g+SJl/ykE8O1HDydAeL9CIppdQg4lDXjYjMF5H9InJIRJ7s4flIEVkrIrkiUiAiy+yeixKR1SKyT0QKRcSxTVxdYG1uBTFhQVw9KtZTJSillNv1GvQi4g+8BCwAsoB7RKT7spAPAXuNMZOBucDzIhJke+4FYKMxZhwwGSh0Uu19cqqtgw8LjzE/O5FAf98ZmlBKqd44knjTgUPGmCJjTBvwOrCk2zEGCBcRAYYCdUCHiEQA1wK/BzDGtBlj6p1VfF98WHiMlvZOnW2jlBp0HAn6FKDM7n657TF7vwHGAxXAHuARY0wXMBKoAf4gIrtE5BURCevpTUTkfhHZISI7ampq+vo5erUur4L48CFMz4hx+msrpZQ3cyToe9p01XS7fwuwG0gGpgC/sbXmA4CpwG+NMVcAzcAFffwAxpgVxphpxphp8fHxjlXvoKbWdj7aX8PCiUn4+116D1mllPI1jgR9OZBmdz8Vq+VubxnwlrEcAoqBcbZzy40xW23HrcYKfrd6f281bR1dLJ6c5O63Vkopj3Mk6LcDo0UkwzbAuhRY0+2YUmAegIgkAGOBImNMFVAmImNtx80D9jql8j5Ym1tBSlQIV6RFu/utlVLK43qdR2+M6RCRh4F3AX9gpTGmQEQesD2/HPgx8EcR2YPV1fOEMabW9hLfBlbZviSKsFr/blN/qo1/HKzla3My8NNuG6XUIOTQBVPGmPXA+m6PLbf7vQK4+SLn7gam9b/Ey7Mxv4qOLqNLEiulBi2fn1C+Lq+S9NhQslMuvtCZUkr5Mp8O+pqm02w+XMuiSclYU/yVUmrw8emg35BfSZdBNwBXSg1qPh3063IrGT1sKGMTwz1dilJKeYzPBn1lQwvbjtRpa14pNej5bND/La8SgEWT9CIppdTg5rNBvzavkgnJEYyMv/hGJEopNRj4ZNCX1Z0it6xe584rpRQ+GvRr86yleLTbRimlfDTo1+VWMiUtirSYUE+XopRSHudzQX+45iR7Kxt1to1SStn4XNCvy61EBBZO1G4bpZQCHwt6Ywxr8yq4Kj2GxMhgT5ejlFJewaeCfn91E4eOndRuG6WUsuNTQb82twI/gQXZiZ4uRSmlvIbPBL0xhnV5lczOjCNu6BBPl6OUUl7DoY1HBoKW9k5mjYzl6sw4T5eilFJexWeCPjQogOfunOTpMpRSyuv4TNeNUkqpnmnQK6WUj9OgV0opH6dBr5RSPk6DXimlfJwGvVJK+TgNeqWU8nEa9Eop5ePEGOPpGi4gIjVAST9PjwNqnVjOQKCf2fcNts8L+pn7aoQxJr6nJ7wy6C+HiOwwxkzzdB3upJ/Z9w22zwv6mZ1Ju26UUsrHadArpZSP88WgX+HpAjxAP7PvG2yfF/QzO43P9dErpZQ6ny+26JVSStnRoFdKKR/nM0EvIvNFZL+IHBKRJz1dj6uJSJqIfCQihSJSICKPeLomdxERfxHZJSLrPF2LO4hIlIisFpF9tn/fszxdk6uJyGO2/67zReQ1EQn2dE3OJiIrReSYiOTbPRYjIu+LyEHbbbQz3ssngl5E/IGXgAVAFnCPiGR5tiqX6wD+xRgzHpgJPDQIPvMZjwCFni7CjV4ANhpjxgGT8fHPLiIpwHeAacaYbMAfWOrZqlzij8D8bo89CXxojBkNfGi7f9l8IuiB6cAhY0yRMaYNeB1Y4uGaXMoYU2mM+dz2exPW//wpnq3K9UQkFVgIvOLpWtxBRCKAa4HfAxhj2owx9R4tyj0CgBARCQBCgQoP1+N0xphNQF23h5cA/237/b+B253xXr4S9ClAmd39cgZB6J0hIunAFcBWD5fiDr8GHge6PFyHu4wEaoA/2LqrXhGRME8X5UrGmKPAfwClQCXQYIx5z7NVuU2CMaYSrMYcMMwZL+orQS89PDYo5o2KyFDgTeBRY0yjp+txJRFZBBwzxuz0dC1uFABMBX5rjLkCaMZJf857K1u/9BIgA0gGwkTkXs9WNbD5StCXA2l291PxwT/1uhORQKyQX2WMecvT9bjBbOA2ETmC1T13g4i86tmSXK4cKDfGnPlrbTVW8PuyG4FiY0yNMaYdeAu42sM1uUu1iCQB2G6POeNFfSXotwOjRSRDRIKwBm7WeLgmlxIRweq3LTTG/MrT9biDMeZfjTGpxph0rH/HfzfG+HRLzxhTBZSJyFjbQ/OAvR4syR1KgZkiEmr773wePj4AbWcN8H9sv/8f4B1nvGiAM17E04wxHSLyMPAu1gj9SmNMgYfLcrXZwFeAPSKy2/bYU8aY9Z4rSbnIt4FVtkZMEbDMw/W4lDFmq4isBj7Hml22Cx9cDkFEXgPmAnEiUg48AzwHvCEiX8P6wvuiU95Ll0BQSinf5itdN0oppS5Cg14ppXycBr1SSvk4DXqllPJxGvRKKeXjNOiVUsrHadArpZSP+//ArEmQrosRPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history1.history['accuracy'], label='train')\n",
    "plt.plot(history1.history['val_accuracy'], label='valid')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD5CAYAAAAp8/5SAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA8X0lEQVR4nO3dd3hUZdrH8e+dRgihhtASIKHX0CKEDtJFAQUVBXbXtaGIiBVddXV9194VRATLAoIIFlCaiAgIAULvEEpIaAkBAgFC2vP+cSYaMZAJTHImk/tzXVwyM+ec+QXhnjPPec79iDEGpZRSnsvL7gBKKaUKlxZ6pZTycFrolVLKw2mhV0opD6eFXimlPJwWeqWU8nA+zmwkIn2B9wBvYLIx5tVLXh8GPOV4mAo8YIzZ7HhtLHAPYICtwF3GmLQrvV/lypVNWFhYAX4MpZQq2davX3/CGBOc12v5FnoR8QbGA72ABGCdiMw1xuzItdkBoKsx5pSI9AMmAe1EJAR4GGhijLkgIrOAocDnV3rPsLAwYmJinPjRlFJKAYhI3OVec2bopi0Qa4zZb4xJB2YCA3NvYIxZZYw55XgYDYTmetkHKC0iPkAAcKQg4ZVSSl0bZwp9CBCf63GC47nLuRtYAGCMOQy8CRwCjgIpxpjFVxdVKaXU1XCm0Esez+XZN0FEumMV+qccjytinf2HAzWAMiIy/DL73iciMSISk5SU5Ex2pZRSTnDmYmwCUDPX41DyGH4RkQhgMtDPGJPseLoncMAYk+TY5hugAzDt0v2NMZOwxvaJjIzUBjxKKadlZGSQkJBAWtoV53l4BH9/f0JDQ/H19XV6H2cK/TqgvoiEA4exLqbemXsDEakFfAOMMMbsyfXSISBKRAKAC0APQK+yKqVcKiEhgbJlyxIWFoZIXoMQnsEYQ3JyMgkJCYSHhzu9X75DN8aYTOAhYBGwE5hljNkuIiNFZKRjs+eBIGCCiGwSkRjHvmuA2cAGrKmVXjjO2pVSylXS0tIICgry6CIPICIEBQUV+JuLU/PojTHzgfmXPDcx1+/vwZorn9e+/wb+XaBUSilVQJ5e5HNczc/pUXfGfvDzXrYdTrE7hlJKuRWPKfSnzqUzY+0hhkxcxfebDtsdRylVgpw+fZoJEyYUeL8bbriB06dPuz7QJTym0Fcs48fc0Z2ICK3AmJmbeOmHHWRmZdsdSylVAlyu0GdlZV1xv/nz51OhQoVCSvUHjyn0AJUDSzH9nnb8o0MYU1Ye4G+fruXkuXS7YymlPNy4cePYt28fLVu25LrrrqN79+7ceeedNG/eHIBBgwbRpk0bmjZtyqRJf8xHCQsL48SJExw8eJDGjRtz77330rRpU3r37s2FCxdcls+pi7HFia+3Fy8MaEqzkPI88+1WbvpgJR+PaEOzkPJ2R1NKFYEX521nx5EzLj1mkxrl+PdNTS/7+quvvsq2bdvYtGkTy5Yto3///mzbtu33KZCffvoplSpV4sKFC1x33XUMHjyYoKCgPx1j7969zJgxg08++YTbbruNOXPmMHx4nveXFphHndHnNqRNKLNHtscYw+CPVvHdRh23V0oVjbZt2/5pnvv7779PixYtiIqKIj4+nr179/5ln/DwcFq2bAlAmzZtOHjwoMvyeNwZfW4RoRWYO7oTo6Zv4JGvNrH1cApP92uEj7fHfr4pVeJd6cy7qJQpU+b33y9btowlS5awevVqAgIC6NatW57z4EuVKvX77729vV06dOPxFa9yYCmm5Rq3HzFlLcmpF+2OpZTyIGXLluXs2bN5vpaSkkLFihUJCAhg165dREdHF3G6ElDo4Y9x+7dubcH6Q6cY8OFvOt8eOJpygdSLmXbHUKrYCwoKomPHjjRr1ownnnjiT6/17duXzMxMIiIieO6554iKiiryfGKM+/UPi4yMNIW18MjWhBTunxpD8rl0Xh3cnJtbhea/k4dJOZ/BO0v2MDU6jg51g5h6dzu7Iyl1TXbu3Enjxo3tjlFk8vp5RWS9MSYyr+1LxBl9bs1DyzNvdCda1arA2K828595JWe+fVa2YfqaOLq/tYz/rT5IRGh5Vuw9wW+xJ+yOppQqRCWu0AMEBZZi6t3tuKtjGJ/+doDhU9Z4/Lj92gMnuemDlfzr223UqxLID6M7M+PeKGqU9+f1Rbtxx292SinXKJGFHqxx+3/f1JS3b2vBxkOnPXbc/sjpC4yesZHbPl7N6fPpfHhnK766L4omNcrh7+vNIz0bsDn+NIu2H7c7qlKqkJTYQp/jltahzB7ZAYDBH63imw0JNidyjbSMLN7/eS/Xv7WMxduPMaZHfX5+rBs3RtT4U/e7W1qHUDe4DG8t3k1Wtp7VK+WJSnyhB2vcfu5DHWlVqwKPztrMi/O2k1FMx+2NMSzYepQeb/3K2z/toUejqvz8WFfG9mpAaT/vv2zv4+3FY70bsjcxlW/1pjKlPJIWeoegwFJMu7sd/+wYzme/HWTElDWcKGbj9ruOneHOT9bwwPQNlPX3Yca9UYwf1prQigFX3K9fs2o0DynPOz/t4WLmlZswKaWKHy30ufh4e/H8TU3+GLf/YCVbE9x/3P70+XSe/34bN7y3gp3HzvDSoGb8MLoT7esG5b8z1kIGT/RpyOHTF5ix5lAhp1VKAQQGBgJw5MgRhgwZkuc23bp1wxVTzbXQ5+GW1qHMeaADIsLgiauYs949x+0zs7KZuvog3d5cxrToOIZH1WbZ490YEVW7wG0eOtevTFSdSnz4Syzn9CYqpYpMjRo1mD17dqG+hxb6y2gWYo3bt6lVkce+3swLc91r3H71vmRu/GAlz32/ncbVyjF/TGf+M7AZFQL8rup4IsKTfRtxIjWdz3474OK0Snm+p5566k896V944QVefPFFevToQevWrWnevDnff//9X/Y7ePAgzZo1A+DChQsMHTqUiIgIbr/9dpf1u3GqqZmI9AXeA7yBycaYVy95fRjwlONhKvCAMWaz47UKwGSgGWCAfxpjVrskfSGz5tu35ZUFu5iy8gA7j55h/LDWVA4slf/OhSTh1Hlemb+LH7ceJaRCaT4a1pq+zaq5ZL3M1rUq0rNxVT5evp/hUbWv+kNDKVstGAfHtrr2mNWaQ79Xr7jJ0KFDeeSRR3jwwQcBmDVrFgsXLmTs2LGUK1eOEydOEBUVxYABAy777/Wjjz4iICCALVu2sGXLFlq3bu2S+Pme0YuINzAe6Ac0Ae4QkSaXbHYA6GqMiQBeAibleu09YKExphHQAtjpiuBFxcfbi+dubMI7t7dgU7w1br8l4XSR57iQnsU7P+2hx1u/8vOu4zzaqwE/P9aVfs2ru3RR5Cf6NCT1YiYf/brPZcdUqiRo1aoViYmJHDlyhM2bN1OxYkWqV6/OM888Q0REBD179uTw4cMcP375e1aWL1/+ew/6iIgIIiIiXJLNmTP6tkCsMWY/gIjMBAYCO3I2MMasyrV9NBDq2LYc0AX4h2O7dKBYLvl0c6tQ6lcpy/1T1zNk4mpeubk5g9sUfp8cYww/bj3Kyz/u5EhKGje1qMHT/RpRo0LpQnm/htXKMqhlCJ//dpB/dgynajn/QnkfpQpNPmfehWnIkCHMnj2bY8eOMXToUKZPn05SUhLr16/H19eXsLCwPFsU5+bKE7cczozRhwDxuR4nOJ67nLuBBY7f1wGSgM9EZKOITBaRMnntJCL3iUiMiMQkJSU5EavoNQux+uRE1i6acfsdR84wdFI0D325kQoBfsy6vz0f3NGq0Ip8jrE9G5BtDO///NfFEZRSlzd06FBmzpzJ7NmzGTJkCCkpKVSpUgVfX19++eUX4uLirrh/ly5dmD59OgDbtm1jy5YtLsnlTKHP6+Mlz1soRaQ7VqHPGa/3AVoDHxljWgHngHF57WuMmWSMiTTGRAYHBzsRyx6Vyvjxv3+25e5O4Xy+6iDDJrt+vv3Jc+n869ut3PjBCvYcP8vLNzdn3uhOtA2v5NL3uZxaQQHc0bYWX62L5+CJc0Xynkp5gqZNm3L27FlCQkKoXr06w4YNIyYmhsjISKZPn06jRo2uuP8DDzxAamoqERERvP7667Rt29YlufJtUywi7YEXjDF9HI+fBjDGvHLJdhHAt0A/Y8wex3PVgGhjTJjjcWdgnDGm/5XeszDbFLvSdxsP89ScLVQq48fHI9oQEVrhmo6XmZXNtOg43lmyl9SLmYyIqs3Yng0oH+DrmsAFkHg2ja6vL6NXk6q8f0erIn9/pQpC2xRfe5vidUB9EQkXET9gKDD3kjeoBXwDjMgp8gDGmGNAvIg0dDzVg1xj+8XdoFYhzHmgA14iDJm4mtnXMN/+t9gT3PD+Cl6Yt4PmIeVZMKYzLwxoakuRB6hS1p+7OoYxd/MRly+0rJQqWvkWemNMJvAQsAhrxswsY8x2ERkpIiMdmz0PBAETRGSTiOQ+HR8NTBeRLUBL4GVX/gB2yz1u//hVjNvHnzzPyKnrGTZ5DRcysvh4RBum3t2WBlXLFmJq59zfpS7l/H14c/Fuu6Mopa6BU/PojTHzgfmXPDcx1+/vAe65zL6bgDy/TniKnHH7VxfsYvLKA+w4eobxd7YmuOzl59ufT8/ko2X7+Hj5frwdLQju7hSOv+9fG4/ZpXyALyO71eX1hbuJOXiSyLCiuUag1NUwxhTKjBV3czVrR+idsS7i4+3Fszc24b2hLdmScJoBH65kc/zpv2xnjOH7TYfp8davfLA0ln7NqrH08a6M6l7PrYp8jrs6hBNcthSvL9TFSZT78vf3Jzk52eP/jhpjSE5Oxt+/YNOenTqjV84b2DKEusGB3D91Pbd+vJr/DmrGrZE1Adh2OIUX521n3cFTNAspxwd3tHL7s+TSft48fH09nvt+O8v2JNG9YRW7Iyn1F6GhoSQkJOCuU7Ndyd/fn9DQgt3DU+IWBy8qJ8+lM3rGBn6LTWZEVG0ys7OZuS6eSgF+PNGnIbdG1sTbq3h8zUzPzKbH28soW8qXH0Z3wquY5FaqJNHFwW1QqYwfX9zVlns7hzM1Oo6vYxL4Z8dwlj7ejaFtaxWbIg/g5+PFo70asOPoGX7cetTuOEqpAtIz+iKwat8Jqpbzp25woN1RrlpWtuGG91aQnpXN4rFd8C1gG2SlVOHSM3qbdahbuVgXeQBvL+HxPg05cOLcNd0voJQqelroldN6Nq5C61oVeG/JXtIydMlBpYoLLfTKadaSg404diaNqauv3JxJKeU+tNCrAmlfN4jO9SszYVksZ9Iy7I6jlHKCFnpVYE/2acSp8xlMXr7f7ihKKSdooVcF1jy0PP2bV2fyygMub9GslHI9LfTqqjzauwEXM7MZ/0us3VGUUvnQQq+uSt3gQIa0DmV69CESTp23O45S6gq00KurNqZnfRB4b4kuOaiUO9NCr65ajQqlGRFVmzkbEohNPGt3HKXUZWihV9fkwW51Ke3rzVuL9+S/sVLKFlro1TUJCizFPZ3rsGDbMbYknLY7jlIqD1ro1TW7p3M4FQN8eWORLjmolDtyqtCLSF8R2S0isSIyLo/Xh4nIFsevVSLS4pLXvUVko4j84Krgyn2U9fdlVPd6rNh7glX7TtgdRyl1iXwLvYh4A+OBfkAT4A4RaXLJZgeArsaYCOAlYNIlr4/BWlhceajhUbWpXt5flxxUyg05c0bfFog1xuw3xqQDM4GBuTcwxqwyxpxyPIwGfl/nSkRCgf7AZNdEVu7I39ebMT3qsyn+ND/tOG53HKVULs4U+hAgPtfjBMdzl3M3sCDX43eBJ4HsgoZTxcuQNqHUqVyGNxfvJitbz+qVchfOFPq81rzL81+xiHTHKvRPOR7fCCQaY9bn+yYi94lIjIjElIQFfj2Rj7cXj/ZuwJ7jqXy38bDdcZRSDs4U+gSgZq7HocCRSzcSkQis4ZmBxphkx9MdgQEichBryOd6EZmW15sYYyYZYyKNMZHBwcEF+BGUO7mhWXWa1ijHO0v2kJ6pX+KUcgfOFPp1QH0RCRcRP2AoMDf3BiJSC/gGGGGM+f3OGWPM08aYUGNMmGO/pcaY4S5Lr9yOl5fwRJ+GJJy6wIy1h+yOo5TCiUJvjMkEHgIWYc2cmWWM2S4iI0VkpGOz54EgYIKIbBIRz1nZWxVY1wbBtAuvxAdLYzmfnml3HKVKPHHHqXCRkZEmJkY/K4qz9XGnGPzRKp7o05BR3evZHUcpjyci640xkXm9pnfGqkLRpnZFejauwsRf93H6fLrdcZQq0bTQq0LzeJ+GpF7MZOKvuuSgUnbSQq8KTaNq5RjYogafrzpA4pk0u+MoVWJpoVeFamyvBmRmGd5fqouTKGUXLfSqUNUOKsPQtjWZuTaeuORzdsdRqkTSQq8K3cPX18fHW3jnJ12cRCk7aKFXha5KOX/+0SGc7zcfYdexM3bHUarE0UKvisQDXesSWMqHN3VxEqWKnBZ6VSTKB/gysmtdluxMZH3cSbvjKFWiaKFXReaujmFUDizFa7o4iVJFSgu9KjIBfj6Mvr4eaw+c5Nc92opaqaKihV4VqTva1iK0YmneWLSbbF2cRKkioYVeFSk/Hy/G9mzA9iNnmL/tqN1xlCoRtNCrIjeoVQgNqgby9uI9ZGbp4iRKFTYt9KrIeXsJj/duyP4T55i9PsHuOEp5PC30yha9mlSlVa0KvPfzXtIysuyOo5RH00Jf2I5sgjn3wrY5didxKyLWkoNHU9KYFh1ndxylPJqP3QE81uH18OvrsGeh9XjnPKjWAirraks5OtStTOf6lRn/Syy3X1eTsv6+dkdSyiM5dUYvIn1FZLeIxIrIuDxeHyYiWxy/VolIC8fzNUXkFxHZKSLbRWSMq38AtxO/DqYNgU+uh/g1cP1zMGod+PrDt/dBlq6hmtsTfRpy6nwGk1ccsDuKUh4r30IvIt7AeKAf0AS4Q0SaXLLZAaCrMSYCeAmY5Hg+E3jMGNMYiAJG5bGvZzgUDVNvhik94cgG6PkCPLIVujwOwQ3gxness/wVb9md1K1EhFagX7NqTF6xn+TUi3bHUcojOXNG3xaINcbsN8akAzOBgbk3MMasMsaccjyMBkIdzx81xmxw/P4ssBMIcVV4t3DwN/hiAHzaB45thV4vwZgt0GkslCr7x3ZNb4bmt8Gvr1kFX/3usd4NuJCRxYRl++yOopRHcqbQhwDxuR4ncOVifTew4NInRSQMaAWsKUA+92QMHFgOn98In98ASbugz8tWge/4MJQKzHu/G96AstXgm/sh/XzRZnZj9aqUZXDrUKZGx3H49AW74yjlcZwp9JLHc3neuy4i3bEK/VOXPB8IzAEeMcbk2ZBcRO4TkRgRiUlKctM+KMbAvl/gsxvgi5vgxF7o+xqM2QztR4FfwJX3L10BBk2A5L2w5IWiSFxsPNKrARh4b4kuTqKUqzlT6BOAmrkehwJHLt1IRCKAycBAY0xyrud9sYr8dGPMN5d7E2PMJGNMpDEmMjg42Nn8RcMYiF1iDc9MHQSnDsINb1oFPmok+JZ2/lh1ukHUg7D2Y9i3tJACFz8hFUozLKoWs9cnEJuYanccpTyKM4V+HVBfRMJFxA8YCszNvYGI1AK+AUYYY/bkel6AKcBOY8zbrotdRIyBPYthck+YNhjOHIH+b8OYTdD2XmsmzdXo8TxUbgjfjYILp/LfvoQY1b0e/r7evP2TLk6ilCvlW+iNMZnAQ8AirIups4wx20VkpIiMdGz2PBAETBCRTSIS43i+IzACuN7x/CYRucH1P4aLGQO7F8An3eHLW+FcItz0HozeANfdDT6lru34vqXhlknWcX983DWZPUDlwFLc0ymc+VuPsTUhxe44SnkMcccFICIjI01MTEz+G7qaMbDrR2tmzLEtUDEMOj8OLYaCdyHczLP8DVj6fzB4CjQf4vrjF0Nn0jLo+vovGGBEVG3+1j6M4LLX+MGqVAkgIuuNMZF5vqaFHsjOhl3z4Nc34PhWqFQHujwBzW8tnAKfIysTPusLJ/bAg9FQrkbhvVcxsv1ICu8t2ctPO4/j6+3FzS1DuKdzOPWrls1/Z6VKKC30l5OdBTu+t86sE3dAUH2rwDcbDN5F1B0ieR9M7AQ128Hwb8BL2w/l2J+UypSVB5i9PoGLmdlc36gK93QOp32dIKzLP0qpHFroL5WdBdu/tXrRnNhtXRjt+qR1U5OXd+G97+XEfAo/jIV+b0C7+4r+/d1ccupFpkUf4n+rD5J8Lp1mIeW4t3MdbmheHV9v/WBUCrTQ/yEr0+oiufwNay57cGOrwDcZaE+Bz2EMfHmbdRPW/SuslgnqL9Iysvh242Emr9jPvqRz1Cjvz10dwxnaVhuiKaWFPisTts6C5W/CyX1QtZlV4Bvd5D5DJWePw4QoqFgb7v6pcK8NFHPZ2YZfdifyyYr9RO8/SdlSPgxtW5O7OoZTo0IB7mlQyoOU3EKflQGbZ8KKN62bnKpFQNenoOEN7lPgc9sxF2aNsDJ2f8buNMXC1oQUPlmxnx+3HkWA/hHVubdzHZqFlLc7mlJFquQV+sx02Pyl1Sny9CGo0coqng36grtfxPt2JGyZBXcvhtA8/5+pPCScOs/nvx1kxtpDnEvPokPdIO7tXIeuDYLx8nLz/+dKuUDJKfSZF2HjVFjxDpxJgJA20HUc1O/l/gU+R1oKfNQRvP1g5ArwK2N3omIl5UIGM9ce4rPfDnLsTBr1qwRyT+dwBrYMwd/XxuswShWyklHo01JgQgerwIe2hW5PQd0exafA53ZghdU07bq7ob/2r78a6ZnZ/Lj1CJ8sP8COo2eoHFiKv7evzfCo2lQs42d3PKX+4qcdx9mScJrR19fHz6fgQ8slo9CDdZdp7Y5W47DiWOBzW/QvWP0hDJsD9XvanabYMsawal8yk5bv59c9Sfj7enFrm5rc3SmcsMr6bUm5h9Pn0+n1znIqB5Zi7kMdr2racMkp9J4kIw0mdbOanj24GgIq2Z2o2Nt97CyTV+zn+01HyMjOpneTqtzXpQ5tauufrbLXY7M2892mw3w/quNVTyS4UqF3w6knCrA6Y94yCc4nWzdTueEHcnHTsFpZ3ri1BSvHdWdUt3pE7z/J4I9Wc8uE31iw9ShZ2fpnrIreL7sTmbMhgQe61i202WJ6Ru/uVrwNP78It3wCEbfZncajnE/P5OuYBKasPMChk+epVSmAuzuFc2tkKAF+RdQCQ5VoZ9Iy6PPOcgJL+fDDw50o5XP1EwZ06KY4y86yVrRK3AkProLyoXYn8jhZ2YbF248xacV+Nh46TfnSvgyPqsXf24dRpdxVrjmglBOe/mYLX62L55sHO9KyZoVrOpYO3RRnXt5w80QwWfDdA1anTeVS3l5Cv+bV+fbBjsx5oD1RdSoxYdk+Or32C0/O3sye42ftjqg80Mq9J5ixNp57O9e55iKfHz2jLy7WfwHzHoa+r0LUA3an8XgHT5xjysoDfL0+nrSMbLo1DObeznXoUFc7Z6prd+5iJr3fWU4pHy/mj+nskns8dOjGExgDM+6w1pm9fzlUaWR3ohLh1Ll0pq+J4/NVcZxIvUijamUZHlWbQa1CCCyl4/jq6jz//TamRsfx9f3tiQxzzawvLfSeIjURJrSH8iFw9xLw0Rt/ikpaRhbfbzrMF6vi2HH0DIGlfLildQjDo2rTQBdEUQWwZn8yt0+K5q6OYfz7pqYuO64Wek+y8wf4api1xGGP5+xOU+IYY9gYf5ppq+P4YetR0jOzaRteiRFRtenTtNpV3dGoSo4L6Vn0fW85xsDCRzq7dHbXNV+MFZG+IrJbRGJFZFwerw8TkS2OX6tEpIWz+6oCanwjtBwOK9+G+LV2pylxRITWtSry9u0tiX66B0/3a8TRlAuMnrGRDq8u5a3Fuzly+oLdMZWbenPxbuKSz/Pa4IgincKb7xm9iHgDe4BeQAKwDrjDGLMj1zYdgJ3GmFMi0g94wRjTzpl986Jn9PlIOwMTO4KXj7VQSalAuxOVaNnZhl/3JjE9Oo6fdyUiQI/GVRkRVZtO9Spr90wFwPq4UwyZuIph7Wrxf4Oau/z4Vzqjd+YjpS0Qa4zZ7zjYTGAg8HuxNsasyrV9NBDq7L7qKviXg0ET4fP+sPhZuOlduxOVaF5eQveGVejesArxJ88zY+0hvloXz087jhMWFMCwdrW5NTKUCgF6TaWkSsvI4snZm6lRvjTj+jUu8vd3ZugmBIjP9TjB8dzl3A0sKOi+InKfiMSISExSUpITsUq4sI7QYTSs/wz2LLY7jXKoWSmAJ/s2YtXT1/Pe0JYEly3Ff+fvpN3LP/P415vZHH/a7ojKBu8u2cu+pHO8cktzW2ZrOfOOeX3vzHO8R0S6YxX6TgXd1xgzCZgE1tCNE7nU9c9C7M/w/Sh4MBrKBNmdSDmU8vFmYMsQBrYMYefRM0yLjuPbjYeZvT6BiNDyDG9Xm5ta1KC0n/bI93Sb408zafk+bo+sSZcGwbZkcOaMPgGometxKHDk0o1EJAKYDAw0xiQXZF91lXxKWY3P0k7DD2O08Zmbaly9HP+9uTlrnunBfwY2tb7Gz9lCu5eX8NIPO9iflGp3RFVILmZm8eTsLQSXLcUz/Yt+yCaHM4V+HVBfRMJFxA8YCszNvYGI1AK+AUYYY/YUZF91jao1s87sd86z1sdVbqusvy9/ax/Goke68NV9UXRpEMwXqw5y/Vu/MnzyGhZuO0Zmlra48CTjf9nH7uNnefnm5pQv7WtbjnyHbowxmSLyELAI8AY+NcZsF5GRjtcnAs8DQcAEx+3hmcaYyMvtW0g/S8nV/iHYvRAWPGmN3VeoZXcidQUiQrs6QbSrE0Ti2TS+WhvPjLWHGDltPdXK+XNnu1oMva6mNlQr5rYfSWHCL7Hc3CqEHo2r2ppFb5jyFKcOWmvN1mgFf5sLXnrjTnGSmZXN0l2JTI2OY8XeE/h4CX2aVmN4VG2i6lTS/jrFTEZWNgM//I3EsxdZ8miXIplxda3TK1VxUDEM+r1mXZiNngAdHrI7kSoAH28vejetRu+m1Thw4hxfroljVkwCP249Sr0qgQxvV4tb2oRSzt++r//KeR//uo8dR88wcXgbt5hWq2f0nsQYmDkMYpfAfcugahO7E6lrkJaRxbzNR5gWHcfmhBQC/KyZPMOjatG0RuGsRKSu3Z7jZ7nx/ZX0alqV8Xe2LrL31V43JUlqEnzUHgKrwb1LtfGZh9iScJpp0XF8v+kIFzOzaV2rAiPa16Zfs+ouaXGrXCMzK5vBH60i/tQFfhrbhaDAUkX23rrwSEkSGAwDPoDjW2HZK3anUS4SEVqB14e0YM0zPXi2f2NOnc9g7Feb6fDqUl5dsItth1NIy8iyO2aJN2XlATYnpPDCgKZFWuTzo2f0nmruaNg4De5aALWi7E6jXCw72/DbvhNMi47jpx3HyTbgJVCrUgD1q5alfpVAGlQtS70qgdSrEqhn/UVgX1Iq/d5bQbcGwXw8ok2RX0DXoZuS6OJZaxaOCIxcCaW0Z7qnOpaSxvq4U+w5fpbYxFT2HD/LgRPnyMy2/m1LzgdAlbLUrxpIg6qB1K9SlrrBgXpnrotkZRtu+3g1sYmp/DS2iy1TY3XWTUlUqqx11+xn/WDRM9ZwjvJI1cr70z+iOv2p/vtz6ZnZxCWfY8/xVPYmnmWv47/Ldifm8QEQ+KdvAfoBUHBfrDrI+rhTvHVrC7e8/0ELvSerFQUdx8DKd6DhDdCwn92JVBHx8/GyinfVspDrAyAj648PgD3Hz7I3MZW9x8/y654kMrL++ACoWfGvHwD1qugHQF7iks/x+qJddG8YzC2tr9Tv0T46dOPpMtPhk+sh9Zij8VlluxMpN5TzAbD3eOqfvgXsP5H6pw+A0IqlaVClLPWqBtLAMRRUr0pgkS6i4U6ysw13To5m++EzLH60C9XLl7Ytiw7dlGQ+ftYQzqSuMG8M3D7N+herVC6+3l7Uq1KWelXK0i/XmhjWB8B5YhPP/v4tIDYxlRV7T5Ceqy9PaMXSNHCc/devWpYGVQOpGxxIGQ9fQH362kNE7z/Jq7c0t7XI58ez/y8oS9Um0ON5a5GSTdOh1XC7E6liwvoAsM7a+zb74/nMrGziTp5n73HrzH+PYwho5SUfAO3CK/HmrS2oWSnAhvSFK+HUeV6dv5NO9Spz+3U189/BRjp0U1JkZ8P/BsCRTfDAb1Cxtt2JlAfKzMrm0Mnz7Dmeys6jZ/h05QFE4PUhEfRtVj3/AxQTxhj+9ula1sedYtEjXdzig0xvmFJWk7NBE6xhm29HQrbeXKNcz8fbizrBgfRtVo2xvRrw48OdCa9chpHTNvD899s85qaur2MSWLH3BE/3a+QWRT4/WuhLkgq1oN/rcGgVrP7Q7jSqBKgVFMDXIztwb+dw/rc6jpsnrCr2C60cS0njpR930C68EsPaFY9vxlroS5oWQ6HxTbD0/+DYNrvTqBLAz8eLf/Vvwqf/iORYygVu/GAl32xIsDvWVTHG8My3W8nIyua1wRF4eRWPiQ1a6EsaEbjxPfCvAN/cB5kX7U6kSojrG1Vl/pjONAspz6OzNvPYrM2cu5hpd6wC+W7TYZbuSuSJPo0Iq1zG7jhO00JfEpUJgoEfQuJ2mHEHxK+zO5EqIaqXL82X97Tj4R71+WZjAjd9uJKdR8/YHcspiWfTeGHuDlrXqsA/OoTZHadAtNCXVA36QJ+X4fB6mNITPu0Lu360Zueo4sMYa/2BQ2vsTuI0H28vHu3VgOn3tCM1LZOB439jWnQc7jgDMIcxhue+28aFjCxeH9IC72IyZJPDqUIvIn1FZLeIxIrIuDxebyQiq0Xkoog8fslrY0Vku4hsE5EZIuJ+jSBKqvajYOx26PsanDkMM++E8ddBzKeQccHudCo/RzbCFzfBtMHw5W1w4ZTdiQqkQ93KzB/Tmag6QTz73TZGfbmBlAsZdsfK049bj7Jo+3HG9mxAvSqBdscpsHwLvYh4A+OBfkAT4A4RuXTpopPAw8Cbl+wb4ng+0hjTDGuB8KEuyK1cpVQgRI2E0RthyGdWM7QfxsI7zWDZa3Au2e6E6lIpCdb1lUndIHEHdH4c0lJg5bt2JyuwyoGl+Pwf1zGuXyMWbz9O//dXsCn+tN2x/iQ59SLPf7+dFqHlubdzuN1xroozZ/RtgVhjzH5jTDowExiYewNjTKIxZh2Q18exD1BaRHyAAODINWZWhcHbB5rdAvf+Av/4EUIjYdnL8E5T+OFRSN5nd0KVdgaWvAgftIHt30HHR+DhjdDjOYi4DdZMhJTDdqcsMC8vYWTXuswa2R5jYMhHq5i0fB/Z2e4xlPPCvB2cTcvg9SEt8PEunqPdzqQOAeJzPU5wPJcvY8xhrLP8Q8BRIMUYs7igIVUREoGwTnDnVzBqLUTcChunWsVl5jCIX2t3wpInKxPWTYb3W8HKt6HxABgdA71eBH/H2rHd/wUmu1ivKta6VkXmP9yZno2r8vL8Xfzzi3Ukp9o7K2zR9mPM23yE0dfXp2G14rumgzOFPq+rDk591IpIRayz/3CgBlBGRPJstCIi94lIjIjEJCUlOXN4VdiCG1p97B/ZBp0fg4MrYUovmNIbds7Tu2sLmzGwe6G1BvCPj1n/P+79BQZ/Yt38llvF2nDdPVYvo8Rd9uR1gfIBvnw0vDUvDWzKqthkbnh/Bav32TN8ePp8Os9+t40m1cvxQLe6tmRwFWcKfQKQu2NPKM4Pv/QEDhhjkowxGcA3QIe8NjTGTDLGRBpjIoODg508vCoSZatawwOP7oB+b8DZY/DVcPjwOlg3RS/cFoajm60LrTNut87Uh35pDamFtL78Pp0fB98y8PN/ii5nIRARRrQP49tRHSjj58OwydG8u2QPWUU8lPOfH3Zw6lw6b9wagW8xHbLJ4Uz6dUB9EQkXET+si6lznTz+ISBKRALEWkCxB7Dz6qIq2/mVgXb3WePCt34BpSvAj49a4/i/vALnTtidsPhLSbB6EX3cFY5vtz5YH4yGRv3zby9dJgg6jYHdP8Kh6KLJW4ia1ijPvNGdGNQyhHeX7GXY5GiOn0krkvdeuus432w4zAPd6tK0RvkieU+2fwuL/lUoNzE61b1SRG4A3sWaNfOpMea/IjISwBgzUUSqATFAOSAbSAWaGGPOiMiLwO1AJrARuMcYc8WfRLtXFhPGwKHVsOoD2D0ffPyh5Z0QNQoq17M7XfFy8ay1Etjq8dafa9RIa7jMv4BFJv2cNZZfqY61MLyHrD0we30Cz323jdJ+3rx1Wwu6N6xSaO91Ji2D3m8vp1xpH+aN7kQpnyJYVevcCRjfDirUhLuXWJMjCkgXB1eFL2mP1Sht80zISrfOQDuMtpYzVJeXlQkbvrAuop5LgmZDrLUDrqWNdMyn1hTZO2Z61PKRsYmpPPTlBnYdO8t9XerweO+G+Pm4fkhl3JwtzIqJ59sHO9KiZgWXHz9Ps/8JO+bC/cut9SOugrYpVoUvuAEMeB/GboMuT0Dcb/BpH5jcy/oLrBdu/8wY2LMIPupgDX8F1YN7lsKQKde+VkCrEVCpLix5waP+3OtVCeS7UR0Z1q4Wk5bv59aPVxN/8rxL32PF3iRmrovn3i51iq7I7/wBts2Brk9edZHPj57Rq8KRfg42fWmd5Z86CBXDrTtxWw4DP/fv312ojm6xVvs68Ks1xNLrP9DoRtcOs2z/Dr7+Owwc75Eriv245Sjj5mwBgdcGR3BD82tf1CT1YiZ93llOKV8v5j/cGX/fIhiyOX8SJkRBYBVrRpW371UfSs/oVdHzKwNt74XRG+C2/0FAEMx/3Lpwu/S/kFoCp9CmHIZvH4CPu8CxrdbaAA+usdpGu3osvclACGkDv7zskbOi+kdUZ/6YztQJDuTB6Rv417dbr3lRk9cW7OJIygXeGBJRNEUeYNEzcD7Z+kC+hiKfHy30qnB5eVtF554lcNdCqNUelr9hFfx5Y+DEXrsTFr6LZ63+/x+0gW2zrWsXD2+Edvdbi7cXBhHo+aLVw2jtpMJ5D5vVrBTA1/e3574udZi+5hCDxv9GbOLVLWoSvT+ZqdFx3NUhnDa1K7k46WXsWQSbZ0CnsVC9RaG+lQ7dqKJ3Yq81u2TTl5B1ERre4Lhw295jZokA1oXWjVOts+pzidBssONCa1jRZZg2BBLWwpjNULpi0b1vEftlVyKPztpEWkY2Lw1qxpA2oU7veyE9i77vLccYWPhIZwL8Cj7jpcDSUmB8lDWr6v5fwafUNR9Sh26Ue6lcH2561+qc2fUpa873Z/1gck9rbLm4X0A0BvYshokd4YdHrHH4e36GIZ8WbZEH6Plvq0dOMWx4VhDdG1VhwZguRISW5/GvN/PoV5ucXtTkjUW7iUs+z2uDI4qmyIN1jSb1GAwa75Iinx89o1f2Sz8Pm7+0zvJP7reKYeMBEFTXuohbqQ6UC7EWOHd3x7Za/4j3L7Ny93yxcMbgC+Kb+2HHd9b1kvJOtakqtrKyDe//vJf3l+4lPKgMH9zZ6oo3PK2PO8mQiasZ3q42Lw1qVjQh9y2FqTdDxzHWhXgX0Xn0qnjIzrJuvFo9HhJiIDtXM1RvP+sDIKfwV6oDlRy/L1+z8Ma6nXXmiHWRedN0647hrk9B5N325wI4FQcfRkLE7dbKYiXA6n3JjJm5kdMXMni2f2NGRNVGLvmwTcvI4ob3V3AxI5tFY7sQWKoIzuYvnoUJHayz+JErwLe0yw59pUJfRN9TlHKCl7d19tv4JqvonzlsneGfPGD999QB6/cHV0LGuT/2Ey+r2OcU/twfBhXDCnc658VUWPW+dXdwdqY1hbTL4+41Hp7T8GzNRGj/EFRpZHeiQte+bhALxnTmsa838/z321kVm8xrgyMoH/DHzJZ3l+xlf9I5pt7dtmiKPFj3NqTEwz8XubTI50fP6FXxYwykJuYq/pd8GFy60lLZ6rmKf9ifPwxKV7i6DNlZ1oXWpf+1LrQ2vcW60FrJTRemOJcM77WA8C5wx5d2pyky2dmGySv38/rC3VQt588Hd7aida2KbI4/zc0TfuPWNjV5bUhE0YQ5sAK+uBGiHoS+rm8nrUM3qmS5cOqv3wJyPgxSj/1529KV8vgm4PhvmeC8x9b3LoGfnrNWd6rZDnr/F2peVzQ/27VY/oY1zfOfi0pca4qNh04xesZGjqWk8WjvBny38TBnLmSy+NEulPMvvPnrv0s/Z90FjcADqwrlW6YWeqVypJ+z7tT9y5DQfqtzpMm1OLpfoKP4O35VqA27frAuplUMtxb+aDyg+EwJzWl4VjEc/rmw+OR2kZQLGYybs4UF26wP+0//Ecn1jaoWzZsvfBqiJ8Dff4DwzoXyFjpGr1QOvzJQtan161KZ6XD60F+HgxJ3wp6FVrM2/wrQ5xVrzNsdLrQWhF8Z6DbOani2Z6FHNTxzRvnSvkwY1pqvYxI4k5ZRdEX+UDREf2T9nSmkIp8fPaNXyhk5F4dLV7IWVC+usjKs3ipePtYQglcR3epfUmVcgImdrJOIB1dBqcJbjlBvmFLqWnl5W8v3FeciD1Y/lR7PQ9Iu6/Z7VbiWvQLJsTDgvUIt8vnRQq9USdN4gEc3PHMbCeutabet/wZ1r7c1ihZ6pUqaEtDwzHaZF+H7URBYDXr/n91ptNArVSKFd4Z6vWDFW3+970Bdu+VvQNJOuOm9gi8HWQicKvQi0ldEdotIrIiMy+P1RiKyWkQuisjjl7xWQURmi8guEdkpIu1dFV4pdQ1+b3j2jt1JPMvRzbDibWhxBzTobXcawIlCLyLewHigH9AEuENELl3v6iTwMPBmHod4D1hojGkEtAB2XlNipZRrVGtu9b9Z87G1KIq6dlkZ8N0oKFMZ+rxsd5rfOXNG3xaINcbsN8akAzOBgbk3MMYkGmPWARm5nxeRckAXYIpju3RjzGlXBFdKuUD3Z6ybxJa5/pb8EmnlO3B8K/R/GwKKaAETJzhT6EOA+FyPExzPOaMOkAR8JiIbRWSyiJQpYEalVGGpWBuuu9fqupm4y+40xdvx7fDr69YCM41vtDvNnzhT6PO6T9rZu6x8gNbAR8aYVsA54C9j/AAicp+IxIhITFJSCVxPVCm7dH7Mavfws+t6o5c4WZnWLBv/8tDvDbvT/IUzhT4BqJnrcShwxMnjJwAJxpg1jsezsQr/XxhjJhljIo0xkcHBwU4eXil1zcoEWYtg7P7Rul1fFdzqD+DIRrjhDevP0804U+jXAfVFJFxE/IChwFxnDm6MOQbEi0hDx1M9gB1XlVQpVXiiHoDAqvDTv6020Mp5SXvgl1esdRSa3mx3mjzlW+iNMZnAQ8AirBkzs4wx20VkpIiMBBCRaiKSADwKPCsiCY4LsQCjgekisgVoCbjPpWillCWn4Vl8NOxeYHea4iM7yxqy8QuwLsC6aUdQbWqmlLJow7OCWz0eFj0Dt3wCEbfZGkWbmiml8qcNzwomeR/8/BI06AvNb7U7zRVpoVdK/UEbnjknOxvmjrYWrb/xHbcdssmhhV4p9QcR6PUfbXiWn5gpEPcb9H0ZytWwO02+tNArpf4srBPU760Nzy7n1EFrdlLdHtBymN1pnKKFXin1Vz204VmejIG5D4N4WZ0p3XzIJocWeqXUX1Vrpg3P8rLhCzjwK/T+D1Somf/2bkILvVIqb9rw7M9SEmDRsxDWGVr/w+40BaKFXimVN2149gdjYN4jYLJgwAfgVbxKZ/FKq5QqWtrwzLJ5BsT+BD1fgErhdqcpMC30SqnL04ZncOYoLBwHtdpb33CKIS30Sqkri3rAWuS6JDY8MwZ+fNRa7Hvg+GI3ZJOjeKZWShWdktzwbNsc2D0frn8WguraneaqaaFXSuWv1QgIqgc/v2gtslESpCbC/CcgJBKiHrQ7zTXRQq+Uyp+3T8lreDb/CUhPdQzZFO9OnlrolVLOyWl4tuwVz294tuN72PGdNWRVpZHdaa6ZFnqllHNKSsOzc8nw42NQvQV0eNjuNC6hhV4p5byS0PBs4Ti4cBoGTrB69HsALfRKqYLx5IZnuxfA1lnQ5XGr34+H0EKvlCqYas2gxVCInmj1f/EUF05ZbQ6qNoNOj9qdxqWcKvQi0ldEdotIrIiMy+P1RiKyWkQuisjjebzuLSIbReQHV4RWStms+zOA8ayGZ4uehXNJ1iwbHz+707hUvoVeRLyB8UA/oAlwh4g0uWSzk8DDwJuXOcwYYOc15FRKuZMKtaDtfbDpS89oeLZ3CWyaBp0egRot7U7jcs6c0bcFYo0x+40x6cBMYGDuDYwxicaYdUDGpTuLSCjQH5jsgrxKKXfhKQ3P0s7AvIehckPo8qTdaQqFM4U+BIjP9TjB8Zyz3gWeBLILsI9Syt0FVPKMhmc/PQdnj8KgCeDrb3eaQuFMoc9rrSynOhuJyI1AojFmvRPb3iciMSISk5SU5MzhlVJ2K+4Nz/Yvg/WfQ/tREBppd5pC40yhTwByr5kVChxx8vgdgQEichBryOd6EZmW14bGmEnGmEhjTGRwcLCTh1dK2ao4Nzy7mApzR0OlutD9X3anKVTOFPp1QH0RCRcRP2AoMNeZgxtjnjbGhBpjwhz7LTXGDL/qtEop91NcG579/CKcjrdm2fiWtjtNocq30BtjMoGHgEVYM2dmGWO2i8hIERkJICLVRCQBeBR4VkQSRKRcYQZXSrkJbx/rJqri1PAsbpXVxqHd/VC7vd1pCp0YNxxXi4yMNDExMXbHUEo5yxiY3BPOHIGHN7j3GXL6eZjYEbKz4MHV1vCTBxCR9caYPC80+BR1GKWUBxKBXi/C5/2tM+WOY+xOZMlMh5P74cRuSNoDJ/bA0U3Wc3+f5zFFPj9a6JVSrpG74Vnrv0HpikX33mln4MReq6Cf2OMo6rvh5AEwWX9sV74mVK5vzbIJ71J0+WymhV4p5To9/g0TO1kNz3q5+EYqYyD1uKOQ7871371wNtdEQC8fayZNlcbQZBAEN7SKe1B9KBXo2kzFhBZ6pZTr5G541vY+KB9a8GNkZ8Gpg38u6Dln6RdT/tjOr6xVwOt0hcoNrF/BDaFimMe0F3YVLfRKKdfq/oy1qPayV6ypi5eTccEx3LLnz0U9ORay0v/YLrCqVcQjbv1zQS9b3bo2oPKlhV4p5Vo5Dc+iJ0D7h6xC/aczc8fvTx/i95vsxcs6E6/cEOr1dAy3OIZcSlew8YfxDFrolVKu1/kx2PA/+LjLn8/OfUpD5XpWu4GWwyDYcYZeqa7H9plxB1rolVKuF1AJBn4IsT/nGm5pAOVrgZeud1TUtNArpQpHk4HWL2U7/WhVSikPp4VeKaU8nBZ6pZTycFrolVLKw2mhV0opD6eFXimlPJwWeqWU8nBa6JVSysO55QpTIpIExF3l7pWBEy6M4yqaq2A0V8ForoLxxFy1jTHBeb3gloX+WohIzOWW07KT5ioYzVUwmqtgSlouHbpRSikPp4VeKaU8nCcW+kl2B7gMzVUwmqtgNFfBlKhcHjdGr5RS6s888YxeKaVULh5T6EWkr4jsFpFYERlnd54cIvKpiCSKyDa7s+QQkZoi8ouI7BSR7SIyxu5MACLiLyJrRWSzI9eLdmfKTUS8RWSjiPxgd5bcROSgiGwVkU0iEmN3nhwiUkFEZovILsfftfZukKmh488p59cZEXnE7lwAIjLW8fd+m4jMEBGXLbnlEUM3IuIN7AF6AQnAOuAOY8wOW4MBItIFSAX+Z4xpZnceABGpDlQ3xmwQkbLAemCQ3X9eIiJAGWNMqoj4AiuBMcaYaDtz5RCRR4FIoJwx5ka78+QQkYNApDHGreaFi8gXwApjzGQR8QMCjDGnbY71O0fdOAy0M8Zc7X07rsoSgvX3vYkx5oKIzALmG2M+d8XxPeWMvi0Qa4zZb4xJB2YCbrG0jTFmOXDS7hy5GWOOGmM2OH5/FtgJhNibCowl1fHQ1/HLLc5ERCQU6A9MtjtLcSAi5YAuwBQAY0y6OxV5hx7APruLfC4+QGkR8QECgCOuOrCnFPoQID7X4wTcoHAVByISBrQC1tgcBfh9eGQTkAj8ZIxxi1zAu8CTQLbNOfJigMUisl5E7rM7jEMdIAn4zDHcNVlEytgd6hJDgRl2hwAwxhwG3gQOAUeBFGPMYlcd31MKveTxnFucCbozEQkE5gCPGGPO2J0HwBiTZYxpCYQCbUXE9uEuEbkRSDTGrLc7y2V0NMa0BvoBoxzDhXbzAVoDHxljWgHnAHe6duYHDAC+tjsLgIhUxBqFCAdqAGVEZLirju8phT4BqJnrcSgu/NrjiRxj4HOA6caYb+zOcynH1/xlQF97kwDQERjgGAufCVwvItPsjfQHY8wRx38TgW+xhjLtlgAk5PpGNhur8LuLfsAGY8xxu4M49AQOGGOSjDEZwDdAB1cd3FMK/TqgvoiEOz6phwJzbc7kthwXPacAO40xb9udJ4eIBItIBcfvS2P95d9layjAGPO0MSbUGBOG9XdrqTHGZWdb10JEyjguqOMYGukN2D7DyxhzDIgXkYaOp3oAtk+OyOUO3GTYxuEQECUiAY5/nz2wrp25hI+rDmQnY0ymiDwELAK8gU+NMdttjgWAiMwAugGVRSQB+LcxZoq9qegIjAC2OsbDAZ4xxsy3LxIA1YEvHLMhvIBZxhi3msrohqoC31q1AR/gS2PMQnsj/W40MN1x8rUfuMvmPACISADWDL377c6SwxizRkRmAxuATGAjLrxL1iOmVyqllLo8Txm6UUopdRla6JVSysNpoVdKKQ+nhV4ppTycFnqllPJwWuiVUsrDaaFXSikPp4VeKaU83P8DCIUdJf8KwTUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history1.history['loss'][2:], label='train')\n",
    "plt.plot(history1.history['val_loss'][2:], label='valid')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the training error is much higher than validation. While this partly has to do with the fact that the validation error is calculated after an epoch, the gap still indicates that the model is severely underfitting. We will first address this by increasing the model's complexity. Clearly, the resizing to 32x32 causes a lot of data loss and adds bias. We also add an extra set of conv/pooling layers to account for the larger image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "reshape_1 (Reshape)          (None, 85, 129, 1)        0         \n",
      "_________________________________________________________________\n",
      "normalization (Normalization multiple                  259       \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 83, 127, 32)       37184     \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 81, 125, 64)       18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 40, 62, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 40, 62, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 40, 62, 32)        2080      \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 38, 60, 32)        9248      \n",
      "_________________________________________________________________\n",
      "conv2d_6 (Conv2D)            (None, 36, 58, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 18, 29, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 18, 29, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 33408)             0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               4276352   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 4,362,373\n",
      "Trainable params: 4,362,114\n",
      "Non-trainable params: 259\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model2 = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=SPECTROGRAM_SHAPE),\n",
    "    tf.keras.layers.Reshape((SPECTROGRAM_SHAPE[0], SPECTROGRAM_SHAPE[1], 1)),\n",
    "    norm_layer,\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Conv2D(32, 1, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(128, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.5),\n",
    "    tf.keras.layers.Dense(2)\n",
    "])\n",
    "model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
    "               metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "163/163 [==============================] - 49s 224ms/step - loss: 239.7868 - accuracy: 0.8400 - val_loss: 0.3490 - val_accuracy: 0.8778\n",
      "Epoch 2/20\n",
      "163/163 [==============================] - 20s 120ms/step - loss: 0.2508 - accuracy: 0.9376 - val_loss: 0.1761 - val_accuracy: 0.9349\n",
      "Epoch 3/20\n",
      "163/163 [==============================] - 20s 121ms/step - loss: 0.1760 - accuracy: 0.9479 - val_loss: 0.1745 - val_accuracy: 0.9440\n",
      "Epoch 4/20\n",
      "163/163 [==============================] - 20s 121ms/step - loss: 0.1325 - accuracy: 0.9629 - val_loss: 0.1115 - val_accuracy: 0.9632\n",
      "Epoch 5/20\n",
      "163/163 [==============================] - 20s 121ms/step - loss: 0.1343 - accuracy: 0.9631 - val_loss: 0.1019 - val_accuracy: 0.9656\n",
      "Epoch 6/20\n",
      "163/163 [==============================] - 20s 121ms/step - loss: 0.1053 - accuracy: 0.9693 - val_loss: 0.0812 - val_accuracy: 0.9727\n",
      "Epoch 7/20\n",
      "163/163 [==============================] - 20s 122ms/step - loss: 0.1253 - accuracy: 0.9669 - val_loss: 0.1363 - val_accuracy: 0.9636\n",
      "Epoch 8/20\n",
      "163/163 [==============================] - 20s 122ms/step - loss: 0.1295 - accuracy: 0.9662 - val_loss: 0.1377 - val_accuracy: 0.9639\n"
     ]
    }
   ],
   "source": [
    "history2 = model2.fit(train_dataset, epochs=20, validation_data=valid_dataset, callbacks=tf.keras.callbacks.EarlyStopping(patience=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxC0lEQVR4nO3deXxV1bn/8c+TiUwkgSSEDEDCTAJhCoMilAooM05VBByoc2udbm2t93drB9vaXq+91Wq9DtgqCCKKgiLOQFUCBCGQhChjyAAhTJnIeM76/bGPGEKAQ6adnDzv1+u8cs7Zw3l2SL6srL322mKMQSmllOfysrsApZRSLUuDXimlPJwGvVJKeTgNeqWU8nAa9Eop5eF87C6gIRERESY+Pt7uMpRSqt3YunXrUWNMZEPL2mTQx8fHk5aWZncZSinVbohIzrmWadeNUkp5OA16pZTycBr0Sinl4dpkH31DampqyMvLo7Ky0u5SWpy/vz9xcXH4+vraXYpSygO0m6DPy8ujc+fOxMfHIyJ2l9NijDEcO3aMvLw8EhIS7C5HKeUB2k3XTWVlJeHh4R4d8gAiQnh4eIf4y0Up1TraTdADHh/y3+kox6mUah3tputGKWWzmkrY8wmUF8Hga8A/1O6KlJvaVYveTidPnuS555676O2mT5/OyZMnm78gpVpDbTV8sxbevhP+uy+8MR/eewCeSoQ1v4Bje+2uULlBW/Ru+i7of/KTn5zxvsPhwNvb+5zbrVmzpqVLU6p5OWpg/3rIWAnZq6GyGPzDqOg/i28ipuDsFMrwQ28gaYtg8wvQfyqMvRsSfgDa7dgmadC76ZFHHmHv3r0MGzYMX19fgoODiY6OZvv27WRlZXHVVVeRm5tLZWUl999/P3feeSfw/XQOZWVlTJs2jcsuu4yvvvqK2NhY3n33XQICAmw+MqUApwMOfAGZb2OyViEVx6nxCSY7bAIfdb6UN4/35XCa07VyJZf0/jGPL/gFfXKWw5aX4dUPoFsijL0HhvwIfPXnui2RtngrwZSUFFN/rptdu3YxaNAgAH67OpOsgpJm/czEmBAem5V0zuUHDhxg5syZZGRksG7dOmbMmEFGRsbpIZDHjx+na9euVFRUMGrUKNavX094ePgZQd+3b1/S0tIYNmwY119/PbNnz2bBggUNfl7d41WqRTidVO3/irKtywnc8x4B1ceoEH8+dY7k3ZoxbHAm4/DqRL+oziRGh5AUE0JiTAi7j5Tx5IffUFZVy62XxvPAxB503v0upP4DCjMgMBxGLoRRt0NItN1H2WGIyFZjTEpDy7RF30ijR48+Y5z7008/zcqVKwHIzc1l9+7dhIeHn7FNQkICw4YNA2DkyJEcOHCgtcpViuJTNWQWnKQo+0vC9r1H4onPiDTHcBo/PnMO4xOvWzgSNZ5+sVFMiQ7h/pgQ+kUF08nnzK7Jsb3DmTEkmv/+8BsWfbmfVekFPDp9IlfdNQ/J+cIK/H//D3z5v5B0jdXKjx1hz0EroJ0G/fla3q0lKCjo9PN169bxySefsHHjRgIDA5k4cWKD4+A7dep0+rm3tzcVFRWtUqvqWIwxHCquJLOghKyCEjLzT1Kbv53Rp9Yx0zuVS+Uo1fiQETCa1Lip+CXOIKlHNNO6BuLl5V4fe9cgP/50zRDmjurBr1dl8uAb6by+6SC/nT2MxBuXwvF9sOkF2PYa7FwOPcZagT9wJni3y9hp1/Q77qbOnTtTWlra4LLi4mK6dOlCYGAg2dnZpKamtnJ1qqOqdTjZd7TcCvSCYrIOWeF+4lQ1AyWXWd4becx3E7HmMA5fb05Gj6ck+VpChs1hhH8oTW1nD+0Rxsp7LmV5Wi5/XpvNzGf+zc2XxPPglP6ETnsCfvgr2LYENj0Pb94CoT1g9B0w4mYI6NIs3wN1YRr0bgoPD2fcuHEMHjyYgIAAoqKiTi+bOnUqzz//PMnJyQwYMICxY8faWKnyVKeqa8k+XHq6pZ5VUEz24VKqaq2TpH4+XkyOOMF/R2xmVPk6Qsv3Y8QbSZgAg/8T74EzCQ/s2ux1eXkJc0f3ZOrg7vzPR9/y6sYDrE4v4JfTBnLdiDi8LvkJjLkLvvnACvyPfw3rnoBh82DM3RDRr9lrUmdqlydjO4KOdrzqTMfKqqxAP1TiCvZi9h8tx+n6dQ0N8D19gnRUyAlGlK4jIuc95EgWIBB/GSRdDYlzICiiVWvPyC/msVWZbM05wfCeYfxu9mCGxNW5uOrQDivwd74Jjmrod4UV+H0u1+GZTXC+k7Ea9G1URzvejqrW4STvRAW7Dn0f6pkFxRSWVJ1eJzYsgMSYkDNGvsRyBMl8BzLfhkPp1oo9xlpXrCbOgc7d7TkgF6fTsHJbPn/6IJtj5VXMG92Th68cQFig3/crlR2BtEWw5SXratvIgVbgJ98AfoH2Fd9OadC3Qx3teD2ZMYYjpVXsKypn/9Fy9h8tc30t5+DxU9Q4rN9Bby+hT2QQSTGhVqBHW6F+OhyL8yFzpRXu+Vut92JHWiNbkq6C0Dh7DvA8Sipr+OvH3/LqxhxC/H14+MqB3DCqB951T/rWVkHG25D6LBzeafXdfzc8MzTWvuLbGQ36dqijHa8nKD5Vw746Ib7vaDkHXM9PVTtOr+fn40VCeBAJEUEkRAaREB7EgO6dGdC9M/6+9a6yLi2ErHesIMx1neTvnmy13JOuhi7xrXZ8TZF9uIRfv5vJ5v3HSY4L5bezkxjes97JWGMg5ytIfQ6y3wcvb+uvk7E/gbgG80vVoUHfDnW0420vKqodHDhWfjrM6z6Ol1efXs9LoEfXQCvMI4LoHRFEvOt5TGjA+Ycxlh+FrHet1vuBLwBjXXWadI0V8OF9Wv5AW4AxhlXpBfzh/V0cKa3ihpQe/GLqAMKDO5298vH9sPlFa3hmVQnEjbKGZw6aDd6ed0Oe0soaPs4q5ODxUzwwuX+j9qFB3w51tONtS2pc/eb7j5axr6j8+2AvKqeg+MzrI6JCOrnCPJiEiEDX1yB6dg3Ez+ci5gw8dRyy37Na7vs3gHFAeD9Xy/0a6DawmY/SPmVVtTz96W4WfbGfQD9vfn7lAOaP6XVmd853qkph++vWRVgn9kNIrGt45i3QAiOIWlNljYPPs4+wKr2Az7KPUFXrJD48kI8f+gG+3hc/36QGfTvU0Y63tRljOFxSyf6icvYfs0K8br95rfP734sQfx96Rwaf0SpPcD0P7nQRI5SrT0FJPhTnub7mQ0kenDgAORvBWWN1xXzXco8a7NGjUHYXlvLYqky+2nuMxOgQfjcniZT4c4S30wG7P7K6dfZvAJ8AGDrXauVHDmjdwpugxuHkiz1HWb29gI+yCimrqiUiuBMzk6OZNTSGET3DGn0/Cg16GwQHB1NWVkZBQQH33XcfK1asOGudiRMn8uSTT5KScva/TXs73rbqRHn1WUH+Xd95Rc33/eb+vl7Eh38f4gkRQfSOtFrqXQJ9L/zLV1sFJQVnBnhx/pmvK06cvV1QpNVKTRhvBXzMcI8O9/qMMazZeZjH38/iUHEl14yI5ZFpA+nW2f/cGx3OsIZn7lgOjiroM8nqx+9zOXi1vZnXnU7D5gPHWZVewAc7D3HiVA0h/j5MG2yF+9jeXfFpRAu+Pp3rxkYxMTENhrxqGdW1TpZsymFVegH7j5Zz8lTN6WXeXkKPLgEkRARxSe9wEiKtvvOEiCC6h/ifu9/cUQtlhxsI8Dot8/IjZ2/nH2aNhAmJhR6jrK/fvQ6Ntb76NNA/3YGICDOSo/nhwEj+/tkeXvz3Pj7OLOSBKf255ZJeDQdg98Ew5+8w+TeQ9gpseRGWXAsR/a0Ls4beCH5BZ2/Xiowx7MgrZnV6Ae/tOMThkkoCfL2ZnBjF7KExTOgfcdYcQi1Jg95Nv/zlL+nVq9fp+eh/85vfICJs2LCBEydOUFNTw+OPP86cOXPO2K7urJcVFRUsXLiQrKwsBg0apHPdNCNjDB9mHuaJD7I5cOwUyXGhTB8SfTrIEyKC6NE18Oy+T6fTGsN9KKvhAC/Jh9LDVp95XX7B3wd21OB6AR5nfbU5bNqTQD8ffjF1INeNjOM3q7P4/XtZLN+Sy2/nJDG2d3jDGwVFwA8ehnH3WyeuU5+D9/8DPv0djLwVRt0BYT1a9Ti+LSxldXoBq7fnc/B4OQHeTib27cqMKbFM7NeFQG/AWQuledZXp8PqsnPWWg/xsv6qa2bts+vmg0es8bbNqfsQmPbEORdv27aNBx54gPXr1wOQmJjI2rVrCQsLIyQkhKNHjzJ27Fh2796NiJzuuqkb9E899RQZGRksWrSIHTt2MGLECFJTU7Xrpom2HTzBH97fRVrOCfp1C+bRGYOY2D8SAau75HRw1wvw4jwoPWRdnVmXjz+ExDTQAo/7viXuH9qhulhakzGGj7IK+d3qLPJPVjB7aAz/OWMQUSHn6c6xNoTcTVbg71oNCAyaCV17WyHqqP0+UE+H7HfPa+q9rrfcce7lDkcNNTXVOGtr8TK1eOPEVxznr/VcgqPg5982alPtumkGw4cP58iRIxQUFFBUVESXLl2Ijo7mwQcfZMOGDXh5eZGfn09hYSHduzd8VeKGDRu47777AEhOTiY5Obk1D8Hj5B4/xZ/XZvPejkNEBHfij1cP4fqUOHzSF8Pfn7aCvLbeX01ePtA5xgrsuFFnB3honDWfuoa4bUSEK5O6M6FfJP9Yv5fn1+/l012F3DepHwvHJZx7NJMI9BxrPU4etO5+tW2xNceOl4/r4V3nuW+913WWe/taX30DGljuS4VDyD1ZzYETVRSW1VKLNxEhgcR3C6V3VCi+/v4NfF5DjzrLvX2tRkYLaJ9Bf56Wd0u67rrrWLFiBYcPH2bu3LksWbKEoqIitm7diq+vL/Hx8Q1OT1xXY8+oq+8Vn6rh75/v5l9f5eDlBfdd3pc7f9CHYD9v2PAkfP64FeL9rjgzwENiIbib9cul2rwAP28emtKfa0fE8rvVWfzpg2yWp+Xy29mDuazfBebvCesJVzxuPZrJyVPVfJBxmNXpBaTuO4bTQGJ0CLPGxjAzOZoeXdvutA3tM+htMnfuXO644w6OHj3K+vXrWb58Od26dcPX15fPP/+cnJyc824/YcIElixZwg9/+EMyMjLYsWNHK1XuGaprnbyWmsPTn+6mpLKGH42M46EpA+ge6m/1tX/4qPVne/INMOdZj7ywpiPqFR7Ey7eO4tNdhfx2dRYLXt7E9CHd+X8zEokJa9lbFpZX1fLJrkJWbS9gw+4iahyGhIgg7r28H7OHRtO3W+cW/fzmokF/EZKSkigtLSU2Npbo6Gjmz5/PrFmzSElJYdiwYQwceP6LWu655x4WLlxIcnIyw4YNY/To0a1UeftmjOGDjMP8eW02OcdOMb5fBL+aNojEmBBrBUcNvHsv7FgGY+6BK//YJofZqaaZNCiKcX0jeGHDPp79fA+fZxdx7+V9uX18QrOOYKmqdbDumyJWpRfw6a5CKmucRIf6s3BcArOHxpAUE9Lu/jJ362SsiEwF/gZ4Ay8ZY56ot7wLsAjoA1QCPzbGZLiWhQEvAYMB41q28Xyf5wnj6Juqox3vuWzNOcEf3s/i64MnGRDVmUdnDOIH/SO/X6GmAt68Fb5dC5f/Pxj/c+1f7wByj5/i8fez+DCzkISIIB6blcjEAd0avb9ah5Ov9h5jVXoBH2YeprSylvAgP6YPsca6p/Tq4vbdt+zSpJOxIuINPAtMAfKALSKyyhiTVWe1R4HtxpirRWSga/1JrmV/A9YaY64TET+g7XZkqTYj51g5f1n7De/vPERk5078+dohXDey3qyHFSdh6Y1wcCPMeApG3WZbvap19egayP/dlML6b4v4zapMbn1lC1MSo/j1zES3+8qdTsPWgydYnV7Amp2HOFpWTedOPlyR1J3Zw2IY1ye8WS5kagvc6boZDewxxuwDEJFlwBygbtAnAn8CMMZki0i8iEQBFcAE4FbXsmqg3lg2pb538lQ1z3y2h1c3HsDHy4sHJvfjjvG9Cao/1UBpISy+Foqy4bpF1pQBqsP5Qf9I1j4wnpe/2M8zn+5h8lPr+cnEvtz1g95nzwSK1Q2YWVBijXVPL6CguJJOPl5MHhTFrKExTBwQ2eB27Z07QR8L5NZ5nQeMqbdOOnAN8IWIjAZ6AXGAAygCXhGRocBW4H5jTHljijXGtLu+scZoi9c2tLSqWgevbbROtJZV1XJ9Sg8enNK/4bHTJw7Aq1dBWSHMewP6Tjp7HdVhdPLx5icT+3LVsFj+sGYXf/3kW1Z8nctjM5OYnGjd8nNvURmrthewekcB+4rK8fESJvSP5BdTBzI5Meri5ixqh9w5uoaStX4SPQH8TUS2AzuBbUAt4AuMAH5mjNkkIn8DHgH+66wPEbkTuBOgZ8+eZ32gv78/x44dIzw83KPD3hjDsWPH8PdvmfG0bY0xhvd3HuLPa7PJPV7BD/pH8qvpAxnYPaThDQoz4bVroLYSbl5lTS2gFBATFsCz80Ywb/RRHluVye2vpnFZ3wiOl1eTdagEERibEM7tl/Vm2uDudAnyu/BOPcQFT8aKyCXAb4wxV7pe/wrAGPOnc6wvwH4gGas/PtUYE+9aNh54xBgz43yf2dDJ2JqaGvLy8i44Tt0T+Pv7ExcXh6+vZw8PTDtwnD+s2cW2gycZ2L0zj04fxIS6J1rrO7gJXv8R+AbCTSuhm56sVg2rcTj555cHeG7dHnqGBzF7qDXW/YJX17ZjTb0ydgvQT0QSgHxgLjCv3geEAadcffC3AxuMMSVAiYjkisgAY8w3WCdos2gEX19fEhISGrOpamMOHC3nz2uz+SDjMFEhnfjLdclcOyKu4fnIv7P7Y3jjJmtqgptWQpderVewand8vb24Y0Jv7pjQ2+5S2oQLBr0xplZE7gU+xBpeucgYkykid7uWPw8MAl4VEQdWkNcd/vAzYIlrxM0+YGEzH4NqJ06UV/P0Z7t5bWMOfj5ePDSlP7ePTyDQ7wI/hjtXwMq7rLssLXjLurpVKeW2djOpmWq/KmscvLrxAM98tofyqlpuGNWTB6f0O/+c49/Z/CKseRh6jYMbX7cmE1NKnUUnNVO2MMawesch/rI2m7wTFfxwQCS/mj6I/lFuXDZuDKz/C6z7IwyYDte9Ar6e27+qVEvSoFctYvN+60Rreu5JBkWHsPi25AtPRPUdpxPWPgKb/w+GzoPZz4C3/qgq1Vj626Oa1b6iMv68NpsPMwvpHuLPkz8aytXDY89/orUuRw28cw/sfBMuuRem/F7nrVGqiTToVbM4Xl7N05/uZnFqDp18vPj5Ff257bLeBPhdxFWG1adg+c2w52OY9Bhc9qDOW6NUM9CgV01SWePgn18d4NnP9nCqxsHcUT14YHJ/Ijtf5L1QK07A63MhbzPM+pt1KzilVLPQoFeN4nQaVu8o4C9rvyH/ZAWTBnbjkWkD6efOidb6Sg9bV7se222ddE26qtnrVaoj06BXFy113zH+uGYXO/KKSYoJ4b+vS+bSvm6eaK3v+D547WooK4J5y6HPD5u3WKWUBr1y396iMp74IJuPswqJDvXnqeuHctWw2MbP0314p9WSd9bCLashbmTzFqyUAjTolRtqHE7+tCabf208QICvNw9fOYDbLkto2nSuORvh9RugUzDc+h5EDmi+gpVSZ9CgV+dVXevkvqXbWJt5mHljevLQlP5EBF/kidb6vv0Qlt9i3bD7ppUQ1qN5ilVKNUiDXp1TVa2Dny75mk92HeG/ZiZy22XNMKncjuWw8m7oPsSatyaokX37Sim3adCrBlXWOLjrta2s/7aI389J4qZL4pu+09TnYe0vIX48zH0d/M8x57xSqllp0KuzVFQ7uOPVNL7ce5QnrhnC3NFn3wjmohgDn/8RNvwFBs6Ea1/WeWuUakUa9OoM5VW1/PifW9hy4DhPXjeUa0fGNW2HTid88DBseQmG3wQz/1fnrVGqlelvnDqttLKGha9sYVvuSf56wzDmDItt2g5rq+GduyHjLRh3P0z+rU5poJQNNOgVAMUVNdy8aDOZ+cU8c+Nwpg+JbtoOq8utO0Lt/dQK+MseaJY6lVIXT4NecaK8mpsWbeKbw6U8N38EVyR1b9oOTx23xsjnp1lTDI+4uXkKVUo1igZ9B3esrIr5L21i39FyXrgphR8ObOJt+koOweJr4NgeuP5VGDSreQpVSjWaBn0HdqS0kvkvbuLg8VO8fEsK4/tFNm2Hx/bCa1dZLfoFb0HChGapUynVNBr0HdTh4krmvZjKoeJKXlk4ikv7NPHCpUPpsPhaME5r3prYEc1TqFKqyTToO6D8kxXMezGVo6VVvHrbaEbFd23aDg98CUvnQqcQuPkdiOjXLHUqpZqHBn0Hk3v8FDe+mEpxRQ2v3T6GET27NG2H33wAb94KYT2teWtCmzjuXinV7DToO5ADR8uZ92Iq5dUOltw+huS4sKbtcPtSePenED0U5q+AoPBmqVMp1bw06DuIPUfKmPdiKrVOw+t3jCEpJrRpO9z4HHz4K0j4AcxdAp0acWcppVSr0KDvAL4tLGXei5sAw9I7xjKgexNC2Rj47HH495MwaDZc+xL4NHHaYqVUi9Kg93BZBSUseHkTPl7C63dcQt9uwY3fmdMB7/8HbH3Funn3jKfAqwk3H1FKtQoNeg+2M6+YBS9vItDPm9fvGEtCRFDjd1ZbBW/fCVnvwGUPwaRf67w1SrUTGvQeatvBE9y8aDMh/r4su3MsPboGNm5HxsDBVPjs95DzJVzxOFz6s+YtVinVojToPVDagePc+soWugb5sfTOscSGBVz8TmqrIXMlpD4Hh7aDfxhc9TwMu7G5y1VKtTANeg+zce8xbvvXFrqH+PP6HWPpHnqRN/goK7L64Le8BGWFENEfZv4Vkm8AvyZ0/SilbKNB70G+2H2U21/dQo8ugSy5fQzdQi4i5A9nwKZ/wI43wVEFfSfD2Oeg9+Xg5dVyRSulWpwGvYf4/Jsj3PXaVnpHBLH49jFEBLsx5NHphN0fWt0z+zeAbyAMXwBj7obI/i1ftFKqVWjQe4CPswr56ZKv6RcVzOLbxtAlyO/8G1SVwrYlsOl5OLEfQmKtm4OMuBkCmzjvjVKqzXEr6EVkKvA3wBt4yRjzRL3lXYBFQB+gEvixMSajznJvIA3IN8bMbKbaFfDBzkP8bOk2kmJCePXHYwgN9D33ysf3w+YXYdtrUFUCPcZYwyQHzQLv82ynlGrXLhj0rpB+FpgC5AFbRGSVMSarzmqPAtuNMVeLyEDX+pPqLL8f2AWENFvlilXpBTz4xnaG9QjjlYWjCPFvIKyNsYZFpv4Dst+3LnBKuhrG3ANxI1u/aKVUq3OnRT8a2GOM2QcgIsuAOUDdoE8E/gRgjMkWkXgRiTLGFIpIHDAD+APwULNW34G9tTWPh1ekkxLflUW3jiK4U71/ytoq66bcqc/B4Z0Q0BXGPwSjboeQGHuKVkrZwp2gjwVy67zOA8bUWycduAb4QkRGA72AOKAQ+F/gF8B5J1gRkTuBOwF69uzpRlkd1xtbDvLI2zu5pHc4L92SQqBfnX/GsiOw5WVIexnKiyByEMx6GpKvB99GjKdXSrV77gR9Q9e5m3qvnwD+JiLbgZ3ANqBWRGYCR4wxW0Vk4vk+xBjzAvACQEpKSv39K5fFqTn8v3cymNA/khduGom/r2uumUPpkPo8ZKwARzX0uxLG3gO9J+pUBUp1cO4EfR7Qo87rOKCg7grGmBJgIYCICLDf9ZgLzBaR6YA/ECIii40xC5qh9g7nlS/389vVWUwa2I1n54/A3xvYtdoK+JwvwDfImmxs9F0Q0dfucpVSbYQ7Qb8F6CciCUA+VnjPq7uCiIQBp4wx1cDtwAZX+P/K9cDVov+5hnzjvLBhL39ck82VSVE8c3Uf/NKeh03/BydzILSnNQfN8JsgIMzuUpVSbcwFg94YUysi9wIfYg2vXGSMyRSRu13LnwcGAa+KiAPrJO1tLVhzh/P3z3bz5EffcutAJ/8Vvhzvp5dAdRn0vBSu+D0MmAHeekmEUqphYkzb6w5PSUkxaWlpdpdhO2MMf/34W7ase5f/7Po5SWUbES8fGHwtjL0bYobbXaJSqo0Qka3GmJSGlmkzsI0yNRV8sPQZpu15lYf8cjHOcGTCwzDqNujc3e7ylFLtiAZ9W1N6GLP5RU5tfInptSc5FNgH5xV/x2vIj8D3ImeiVEopNOjbjoJtkPoPTMbbGGctXzlGkD/wVm65cQGis0cqpZpAg95OjlrIfs+aXOzgRoxfMF91vYpf5V/KlZeN5dHpgxAdA6+UaiINervkb4Xlt0BxLoT1wnnFH/n1wWEs3n6Sn0zsw8NXDtCQV0o1Cw16u6z/C9RUwA1LqO17Jf/xVgbvbi/ggcn9uH9SPw15pVSz0c5fO5QVwZ5PYPgCavpP5/7lO3l3ewEPXzmAByb315BXSjUrbdHbIeMtcNZSM/h6frrkaz7KKuQ/pw/ijgm97a5MKeWBNOjtkL4UZ/eh3PXhKT7LPsJvZiVy67gEu6tSSnko7bppbUey4dB21vtfzmfZR3j8qsEa8kqpFqVB39p2LMOIN3/KTWLyoG4sGNvL7oqUUh5Og741OZ2wYzlHul3Gt+WBzNeQV0q1Ag361nTg31CSz9KqS4nrEsCEfpF2V6SU6gA06FtT+jIcfp35x+EBzBvTE28vHUaplGp5GvStpbocdq1ie+eJOL07cX1Kjwtvo5RSzUCDvrVkvw/VZTxzbCRTB0cTEdzJ7oqUUh2EBn1rSV9KWUAs6yv7smBMT7urUUp1IBr0raHkEOxbx/uMp19UCKMTutpdkVKqA9Ggbw073wTj5PmTo5g/ppfOZaOUalU6BUJrSF9GTkAShx1xXD0i1u5qlFIdjLboW9rhnXAkk3+Wj2HOsBhC/H3trkgp1cFo0Le09GU4xIeV1WOYP0avhFVKtT4N+pbkqMXsfJOvvFPoFRfHkLhQuytSSnVAGvQtad86pKyQxacu0XltlFK20aBvSelLKfcKIc0vhVnJMXZXo5TqoDToW0plCSb7fd6pGc3skQkE+HnbXZFSqoPSoG8pu1YhtRWsqB2vJ2GVUrbScfQtxGxfSq5E4x8/hr7dgu0uRynVgWmLviWcPIjkfMGb1eNYcEm83dUopTo4DfqWsGM5AOv9L2dKYpTNxSilOjrtumluxlCzbSlfOwcy4ZIU/Hz0/1KllL00hZpbwdf4ntjDSsd4btTpiJVSbYC26JuZY9tSavGlrM8MYsMC7C5HKaXca9GLyFQR+UZE9ojIIw0s7yIiK0Vkh4hsFpHBrvd7iMjnIrJLRDJF5P7mPoA2pbaa2h0r+MgxkmsvTbK7GqWUAtwIehHxBp4FpgGJwI0iklhvtUeB7caYZOBm4G+u92uB/zDGDALGAj9tYFvPsecTOlWfYEPAJCb0j7S7GqWUAtxr0Y8G9hhj9hljqoFlwJx66yQCnwIYY7KBeBGJMsYcMsZ87Xq/FNgFeOyE7GWbX6PIhNB7zCy8vfTmIkqptsGdoI8Fcuu8zuPssE4HrgEQkdFALyCu7goiEg8MBzY19CEicqeIpIlIWlFRkVvFtykVJ/Df/zHvOy/lR2N6212NUkqd5k7QN9Q0NfVePwF0EZHtwM+AbVjdNtYORIKBt4AHjDElDX2IMeYFY0yKMSYlMrL9dXtUp7+Nj6mhsPc1RAR3srscpZQ6zZ1RN3lAjzqv44CCuiu4wnshgFg3RN3veiAivlghv8QY83Yz1NwmFW96jePOOCZOmGR3KUopdQZ3WvRbgH4ikiAifsBcYFXdFUQkzLUM4HZggzGmxBX6LwO7jDFPNWfhbcrxfUSe2Ma/AyYxune43dUopdQZLhj0xpha4F7gQ6yTqcuNMZkicreI3O1abRCQKSLZWKNzvhtGOQ64CbhcRLa7HtOb/ShsVvjvf+E0QsjoeVj/tymlVNvh1gVTxpg1wJp67z1f5/lGoF8D231Bw338nsMYfDKWk8pgpo4baXc1Sil1Fp0CoYnK9nxJeE0BuXGzCfH3tbscpZQ6iwZ9E+WvW8Qp04nBkxfYXYpSSjVIg74JTE0FMflr2eR/KUkJek9YpVTbpEHfBLu/WEFnyvEaeqPdpSil1Dlp0DdB5dalFNKF0ZdfbXcpSil1Thr0jXS0MJ9BpansiZpOgL/fhTdQSimbaNA3UtbHr+ArDnpM/LHdpSil1Hlp0DeCw2mI2LuS/b596Dkoxe5ylFLqvDToGyFty0YSzR5ODbzO7lKUUuqCNOgb4ehXr1KLF/0mLbS7FKWUuiAN+ouUd7yMYSc/4mDYGPzCou0uRymlLkiD/iJ9+cm7xMoxQsfebHcpSinlFg36i1Bd6yRg15uckkDCR+rYeaVU+6BBfxE+2bGfy50bORk/HXwD7C5HKaXcokF/EfZueINgqSRq/K12l6KUUm7ToHfTniNlDDm2lpJO0XjHj7O7HKWUcpsGvZve/WIr47124D1sLnjpt00p1X5oYrmhotqB2fEm3mIIGjXf7nKUUuqiaNC7YfWOAqY7N1AaMQwizrpjolJKtWka9G748ovPSfTKIVhb80qpdkiD/gJ25hWTdHQtDvFBBl9rdzlKKXXRNOgvYGnqXq72/hJn3ykQFG53OUopddF87C6gLSuprOFI+kdEep+E4fPsLkcppRpFW/TnsfLrfGawgdpOodD/SrvLUUqpRtGgPwdjDG9t3MV07zR8hlwLPp3sLkkppRpFg/4cNu8/Tv/jn9OJKhh6o93lKKVUo2nQn8OSTQe5zvdLnF16Q9wou8tRSqlG06BvwNGyKrZn7GQMmXgNnQsidpeklFKNpkHfgOVpuczkCwQDydfbXY5SSjWJDq+sx+E0vJ6aw7KAjRBzCXRNsLskpZRqEm3R17NhdxFdirOIqz0IQ+faXY5SSjWZBn09S1JzmOf/Fca7EyReZXc5SinVZBr0deSfrGBDdgFzvL9CBkyDgDC7S1JKqSZzK+hFZKqIfCMie0TkkQaWdxGRlSKyQ0Q2i8hgd7dtS5ZtPsgEr3QCa0/q2HmllMe4YNCLiDfwLDANSARuFJHEeqs9Cmw3xiQDNwN/u4ht24Qah5NlW3K5K3QLBEZA30l2l6SUUs3CnRb9aGCPMWafMaYaWAbMqbdOIvApgDEmG4gXkSg3t20TPsospKr0GCMqU2HIdeDta3dJSinVLNwJ+lggt87rPNd7daUD1wCIyGigFxDn5ra4trtTRNJEJK2oqMi96pvRkk05zA/ehrezWkfbKKU8ijtB39Bloabe6yeALiKyHfgZsA2odXNb601jXjDGpBhjUiIjI90oq/nsLSrjq73HuClwI0QOhOhhrfr5SinVkty5YCoP6FHndRxQUHcFY0wJsBBARATY73oEXmjbtmBJ6kF6ex8hpmQ7THpMpzxQSnkUd1r0W4B+IpIgIn7AXGBV3RVEJMy1DOB2YIMr/C+4rd0qaxys2JrLQ923A6JTHiilPM4FW/TGmFoRuRf4EPAGFhljMkXkbtfy54FBwKsi4gCygNvOt23LHErjrE4voKSyhsnVn0PCeAiNs7skpZRqVm7NdWOMWQOsqffe83WebwT6ubttW7J400Fmd83DvzQHJrXpYf5KKdUoHfrK2Iz8YtJzT/KTrmngGwiDZtldklJKNbsOHfRLNuXQ2dfBgKKPYOBM6NTZ7pKUUqrZddigL6ms4Z1tBTwcvx+pKtax80opj9Vhg37l1/lU1DiY4/UFBHeH3hPtLkkppVpEhwx6YwxLNuUwLgZCcz+D5B+Bl7fdZSmlVIvokEG/5cAJvi0s46HoDHDW6kyVSimP1iGDfnFqDp39fRh2Yi1EDYGoJLtLUkqpFtPhgv5oWRUfZBzizkQH3gVf60lYpZTH63A3B38zLY8ah2G+fyqIFwz5kd0lKaVUi+pQLXqn0/D65hzGxofRde9K6HM5dI6yuyyllGpRHSroN+wuIvd4Bff1LYLiXD0Jq5TqEDpU0C9OPUhEsB9jSj8Gv84wYLrdJSmlVIvrMEGff7KCz7ILmTc8Eu9d70LiHPALtLsspZRqcR0m6N/YfBAD3NI1E6pLdbSNUqrD6BBBX+NwsmxLLj8c0I3wvSshtAf0Gmd3WUop1So6RNB/nFXIkdIqFiYHwN5PrbtIeXWIQ1dKqY4R9ItTc4gNC2Bc5TowTkjWbhulVMfh8UG/t6iMr/YeY96YnnjtWAYxIyCyv91lKaVUq/H4oH9900F8vIS5vUrg8E4dO6+U6nA8Ougraxys2JrHlYO7E75nJXj5wOBr7S5LKaValUcH/Xs7DlFcUcOCUXGwYzn0uwKCwu0uSymlWpVHB/3i1Bz6RAYxVnZC2WEdO6+U6pA8Nugz8ovZnnuS+WN6ITveAP9Q6D/V7rKUUqrVeWzQL9l0EH9fL64dEga7VkPSNeDTye6ylFKq1Xlk0JdW1vDu9nxmD40hdP9aqDmlo22UUh2WRwb9ym35nKp2MH9ML0hfCl0SoMdou8tSSilbeFzQG2NYknqQIbGhDA0ph/0brJOwInaXppRStvC4oE/LOcE3haUsGNsTdi4HDCTfYHdZSillG48L+sWpOXT292FWcjSkL4MeY6Frgt1lKaWUbTwq6I+VVfHBzsNcOyKOwGMZUJStY+eVUh2eRwX9m1vzqHY4mT+mJ6S/Ad6dIOkqu8tSSilbeUzQO52G1zcdZHRCV/pF+MPON2HAVAjoYndpSillK7eCXkSmisg3IrJHRB5pYHmoiKwWkXQRyRSRhXWWPeh6L0NEloqIf3MewHcqax1MSYzi9ssSYO9ncOqojp1XSincCHoR8QaeBaYBicCNIpJYb7WfAlnGmKHAROB/RMRPRGKB+4AUY8xgwBtokU7zQD8f/mtmIlckdbfGzgeGQ9/JLfFRSinVrrjToh8N7DHG7DPGVAPLgDn11jFAZxERIBg4DtS6lvkAASLiAwQCBc1S+blUnITsNTD4OvD2bdGPUkqp9sCdoI8Fcuu8znO9V9ffgUFYIb4TuN8Y4zTG5ANPAgeBQ0CxMeajhj5ERO4UkTQRSSsqKrrIw6gj611wVOloG6WUcnEn6Bu6pNTUe30lsB2IAYYBfxeREBHpgtX6T3AtCxKRBQ19iDHmBWNMijEmJTIy0s3yG5C+DCL6Q8zwxu9DKaU8iDtBnwf0qPM6jrO7XxYCbxvLHmA/MBCYDOw3xhQZY2qAt4FLm172OZw4AAe/0ikPlFKqDneCfgvQT0QSRMQP62TqqnrrHAQmAYhIFDAA2Od6f6yIBLr67ycBu5qr+LPsWA4IDLm+xT5CKaXaG58LrWCMqRWRe4EPsUbNLDLGZIrI3a7lzwO/B/4pIjuxunp+aYw5ChwVkRXA11gnZ7cBL7TIkRhjjbaJvwzCelx4faWU6iAuGPQAxpg1wJp67z1f53kBcMU5tn0MeKwJNbqn5pQV8r0ntvhHKaVUe+JW0LcLfkEw+xm7q1BKqTbHY6ZAUEop1TANeqWU8nAa9Eop5eE06JVSysNp0CullIfToFdKKQ+nQa+UUh5Og14ppTycGFN/Ikr7iUgRkNPIzSOAo81YTktqT7VC+6q3PdUK7ave9lQrtK96m1JrL2NMg1P/tsmgbwoRSTPGpNhdhzvaU63QvuptT7VC+6q3PdUK7avelqpVu26UUsrDadArpZSH88Sgb5lpkFtGe6oV2le97alWaF/1tqdaoX3V2yK1elwfvVJKqTN5YoteKaVUHRr0Sinl4Twm6EVkqoh8IyJ7ROQRu+s5HxFZJCJHRCTD7louRER6iMjnIrJLRDJF5H67azofEfEXkc0iku6q97d213QhIuItIttE5D27a7kQETkgIjtFZLuIpNldz/mISJiIrBCRbNfP7yV213QuIjLA9T397lEiIg802/49oY9eRLyBb4EpQB7WDc1vNMZk2VrYOYjIBKAMeNUYM9jues5HRKKBaGPM1yLSGdgKXNWGv7cCBBljykTEF/gCuN8Yk2pzaeckIg8BKUCIMWam3fWcj4gcAFJc94Ru00TkX8C/jTEviYgfEGiMOWlzWRfkyrN8YIwxprEXjp7BU1r0o4E9xph9xphqYBkwx+aazskYswE4bncd7jDGHDLGfO16XgrsAmLtrercjKXM9dLX9WizrRkRiQNmAC/ZXYsnEZEQYALwMoAxpro9hLzLJGBvc4U8eE7QxwK5dV7n0YbDqL0SkXhgOLDJ5lLOy9UVsh04AnxsjGnL9f4v8AvAaXMd7jLARyKyVUTutLuY8+gNFAGvuLrFXhKRILuLctNcYGlz7tBTgl4aeK/NtuLaIxEJBt4CHjDGlNhdz/kYYxzGmGFAHDBaRNpk95iIzASOGGO22l3LRRhnjBkBTAN+6uqGbIt8gBHAP4wxw4FyoE2fuwNwdTHNBt5szv16StDnAT3qvI4DCmyqxeO4+rrfApYYY962ux53uf5UXwdMtbeScxoHzHb1ey8DLheRxfaWdH7GmALX1yPASqxu07YoD8ir89fcCqzgb+umAV8bYwqbc6eeEvRbgH4ikuD6H3EusMrmmjyC6+Tmy8AuY8xTdtdzISISKSJhrucBwGQg29aizsEY8ytjTJwxJh7rZ/YzY8wCm8s6JxEJcp2Qx9UNcgXQJkeOGWMOA7kiMsD11iSgTQ4gqOdGmrnbBqw/b9o9Y0ytiNwLfAh4A4uMMZk2l3VOIrIUmAhEiEge8Jgx5mV7qzqnccBNwE5XvzfAo8aYNfaVdF7RwL9cIxe8gOXGmDY/bLGdiAJWWv/34wO8boxZa29J5/UzYImr8bcPWGhzPeclIoFYIwfvavZ9e8LwSqWUUufmKV03SimlzkGDXimlPJwGvVJKeTgNeqWU8nAa9Eop5eE06JVSysNp0CullIf7/xKOQUQNpqhTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history2.history['accuracy'], label='train')\n",
    "plt.plot(history2.history['val_accuracy'], label='valid')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wow, this did better than I expected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47/47 [==============================] - 2s 41ms/step - loss: 0.1377 - accuracy: 0.9639\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.13773751258850098, 0.9638879299163818]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model2.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model size reduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Right now, the model size is a bit too large, with 4 million parameters. It also takes 2 seconds to evaluate a batch - since each batch is worth 8 seconds of audio, this may become a significant bottleneck for realtime conversion of streamed data. We can reduce the number of parameters by increasing the pooling size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "reshape_2 (Reshape)          (None, 85, 129, 1)        0         \n",
      "_________________________________________________________________\n",
      "normalization (Normalization multiple                  259       \n",
      "_________________________________________________________________\n",
      "conv2d_7 (Conv2D)            (None, 83, 127, 32)       37184     \n",
      "_________________________________________________________________\n",
      "conv2d_8 (Conv2D)            (None, 81, 125, 64)       18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 27, 41, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 27, 41, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_9 (Conv2D)            (None, 27, 41, 32)        2080      \n",
      "_________________________________________________________________\n",
      "conv2d_10 (Conv2D)           (None, 25, 39, 32)        9248      \n",
      "_________________________________________________________________\n",
      "conv2d_11 (Conv2D)           (None, 23, 37, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 7, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 7, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 5376)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               688256    \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 774,277\n",
      "Trainable params: 774,018\n",
      "Non-trainable params: 259\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model3 = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=SPECTROGRAM_SHAPE),\n",
    "    tf.keras.layers.Reshape((SPECTROGRAM_SHAPE[0], SPECTROGRAM_SHAPE[1], 1)),\n",
    "    norm_layer,\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(3),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Conv2D(32, 1, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(3),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(128, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.5),\n",
    "    tf.keras.layers.Dense(2)\n",
    "])\n",
    "model3.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model3.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
    "               metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "163/163 [==============================] - 36s 170ms/step - loss: 62.9791 - accuracy: 0.8411 - val_loss: 0.2315 - val_accuracy: 0.9237\n",
      "Epoch 2/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.2590 - accuracy: 0.9261 - val_loss: 0.2106 - val_accuracy: 0.9281\n",
      "Epoch 3/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.2152 - accuracy: 0.9334 - val_loss: 0.1549 - val_accuracy: 0.9531\n",
      "Epoch 4/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1811 - accuracy: 0.9480 - val_loss: 0.1454 - val_accuracy: 0.9558\n",
      "Epoch 5/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1643 - accuracy: 0.9527 - val_loss: 0.1678 - val_accuracy: 0.9396\n",
      "Epoch 6/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1636 - accuracy: 0.9492 - val_loss: 0.1431 - val_accuracy: 0.9571\n",
      "Epoch 7/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1573 - accuracy: 0.9506 - val_loss: 0.1440 - val_accuracy: 0.9568\n",
      "Epoch 8/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1478 - accuracy: 0.9549 - val_loss: 0.1415 - val_accuracy: 0.9568\n",
      "Epoch 9/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1550 - accuracy: 0.9527 - val_loss: 0.1294 - val_accuracy: 0.9575\n",
      "Epoch 10/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1470 - accuracy: 0.9530 - val_loss: 0.1296 - val_accuracy: 0.9625\n",
      "Epoch 11/20\n",
      "163/163 [==============================] - 17s 107ms/step - loss: 0.1292 - accuracy: 0.9582 - val_loss: 0.1152 - val_accuracy: 0.9646\n",
      "Epoch 12/20\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1350 - accuracy: 0.9598 - val_loss: 0.1229 - val_accuracy: 0.9642\n",
      "Epoch 13/20\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1254 - accuracy: 0.9602 - val_loss: 0.1240 - val_accuracy: 0.9629\n"
     ]
    }
   ],
   "source": [
    "history3 = model3.fit(train_dataset, epochs=20, validation_data=valid_dataset, callbacks=tf.keras.callbacks.EarlyStopping(patience=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "While this did not decrease the performance, it also did not resolve its slowness problem by much. Indeed, it looks like the bottleneck is the size of the convolution layers. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "reshape_3 (Reshape)          (None, 85, 129, 1)        0         \n",
      "_________________________________________________________________\n",
      "normalization (Normalization multiple                  259       \n",
      "_________________________________________________________________\n",
      "conv2d_12 (Conv2D)           (None, 83, 127, 8)        9296      \n",
      "_________________________________________________________________\n",
      "conv2d_13 (Conv2D)           (None, 81, 125, 16)       1168      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2 (None, 27, 41, 16)        0         \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 27, 41, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 25, 39, 32)        4640      \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 23, 37, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2 (None, 7, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 7, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 5376)              0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 128)               688256    \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 722,373\n",
      "Trainable params: 722,114\n",
      "Non-trainable params: 259\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model4 = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=SPECTROGRAM_SHAPE),\n",
    "    tf.keras.layers.Reshape((SPECTROGRAM_SHAPE[0], SPECTROGRAM_SHAPE[1], 1)),\n",
    "    norm_layer,\n",
    "    tf.keras.layers.Conv2D(8, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(16, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(3),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(3),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(128, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.5),\n",
    "    tf.keras.layers.Dense(2)\n",
    "])\n",
    "model4.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model4.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
    "               metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "163/163 [==============================] - 31s 139ms/step - loss: 118.5807 - accuracy: 0.7416 - val_loss: 0.5163 - val_accuracy: 0.7776\n",
      "Epoch 2/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.4689 - accuracy: 0.7938 - val_loss: 0.3709 - val_accuracy: 0.8680\n",
      "Epoch 3/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3655 - accuracy: 0.8425 - val_loss: 0.3414 - val_accuracy: 0.8633\n",
      "Epoch 4/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3411 - accuracy: 0.8449 - val_loss: 0.3275 - val_accuracy: 0.8657\n",
      "Epoch 5/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3287 - accuracy: 0.8518 - val_loss: 0.3166 - val_accuracy: 0.8741\n",
      "Epoch 6/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3205 - accuracy: 0.8610 - val_loss: 0.3056 - val_accuracy: 0.8704\n",
      "Epoch 7/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3226 - accuracy: 0.8619 - val_loss: 0.3013 - val_accuracy: 0.8724\n",
      "Epoch 8/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3129 - accuracy: 0.8673 - val_loss: 0.3005 - val_accuracy: 0.8704\n",
      "Epoch 9/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3051 - accuracy: 0.8709 - val_loss: 0.3020 - val_accuracy: 0.8751\n",
      "Epoch 10/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3045 - accuracy: 0.8732 - val_loss: 0.2931 - val_accuracy: 0.8805\n",
      "Epoch 11/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.3004 - accuracy: 0.8776 - val_loss: 0.2855 - val_accuracy: 0.8829\n",
      "Epoch 12/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2937 - accuracy: 0.8766 - val_loss: 0.2765 - val_accuracy: 0.8859\n",
      "Epoch 13/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2868 - accuracy: 0.8793 - val_loss: 0.2735 - val_accuracy: 0.8896\n",
      "Epoch 14/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2780 - accuracy: 0.8836 - val_loss: 0.2581 - val_accuracy: 0.8934 loss: - ETA: 0s - loss: 0.278\n",
      "Epoch 15/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2664 - accuracy: 0.8881 - val_loss: 0.2530 - val_accuracy: 0.8971\n",
      "Epoch 16/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2622 - accuracy: 0.8903 - val_loss: 0.2532 - val_accuracy: 0.8994\n",
      "Epoch 17/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2532 - accuracy: 0.8974 - val_loss: 0.2368 - val_accuracy: 0.9052\n",
      "Epoch 18/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2465 - accuracy: 0.9009 - val_loss: 0.2293 - val_accuracy: 0.9099\n",
      "Epoch 19/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2390 - accuracy: 0.9038 - val_loss: 0.2300 - val_accuracy: 0.9089\n",
      "Epoch 20/20\n",
      "163/163 [==============================] - 12s 74ms/step - loss: 0.2310 - accuracy: 0.9025 - val_loss: 0.2152 - val_accuracy: 0.9136\n"
     ]
    }
   ],
   "source": [
    "history4 = model4.fit(train_dataset, epochs=20, validation_data=valid_dataset, callbacks=tf.keras.callbacks.EarlyStopping(patience=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sadly, while this improves efficiency it falls short on performance."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There was a lot of stochasticity in training the model. Even with the same parameters there was a large variance in its performance. In order to ensure that we get a good result, we trained for longer epochs, checkpointing when the model outperformed all the previous epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "reshape_7 (Reshape)          (None, 85, 129, 1)        0         \n",
      "_________________________________________________________________\n",
      "normalization (Normalization multiple                  259       \n",
      "_________________________________________________________________\n",
      "conv2d_31 (Conv2D)           (None, 83, 127, 32)       37184     \n",
      "_________________________________________________________________\n",
      "conv2d_32 (Conv2D)           (None, 81, 125, 64)       18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_13 (MaxPooling (None, 27, 41, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_20 (Dropout)         (None, 27, 41, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_33 (Conv2D)           (None, 27, 41, 32)        2080      \n",
      "_________________________________________________________________\n",
      "conv2d_34 (Conv2D)           (None, 25, 39, 32)        9248      \n",
      "_________________________________________________________________\n",
      "conv2d_35 (Conv2D)           (None, 23, 37, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_14 (MaxPooling (None, 7, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "dropout_21 (Dropout)         (None, 7, 12, 64)         0         \n",
      "_________________________________________________________________\n",
      "flatten_7 (Flatten)          (None, 5376)              0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 128)               688256    \n",
      "_________________________________________________________________\n",
      "dropout_22 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 774,277\n",
      "Trainable params: 774,018\n",
      "Non-trainable params: 259\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.Input(shape=SPECTROGRAM_SHAPE),\n",
    "    tf.keras.layers.Reshape((SPECTROGRAM_SHAPE[0], SPECTROGRAM_SHAPE[1], 1)),\n",
    "    norm_layer,\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(3),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Conv2D(32, 1, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
    "    tf.keras.layers.Conv2D(64, 3, activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D(3),\n",
    "    tf.keras.layers.Dropout(0.25),\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(128, activation='relu'),\n",
    "    tf.keras.layers.Dropout(0.5),\n",
    "    tf.keras.layers.Dense(2)\n",
    "])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
    "               metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "163/163 [==============================] - 19s 110ms/step - loss: 98.6932 - accuracy: 0.8169 - val_loss: 0.3526 - val_accuracy: 0.8336\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 2/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.3490 - accuracy: 0.8837 - val_loss: 0.3461 - val_accuracy: 0.8934\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 3/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.2731 - accuracy: 0.9063 - val_loss: 0.1965 - val_accuracy: 0.9349\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 4/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.2090 - accuracy: 0.9304 - val_loss: 0.2589 - val_accuracy: 0.9065\n",
      "Epoch 5/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1778 - accuracy: 0.9422 - val_loss: 0.1511 - val_accuracy: 0.9514\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 6/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1596 - accuracy: 0.9567 - val_loss: 0.1349 - val_accuracy: 0.9463\n",
      "Epoch 7/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1291 - accuracy: 0.9585 - val_loss: 0.0835 - val_accuracy: 0.9673\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 8/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1185 - accuracy: 0.9654 - val_loss: 0.2829 - val_accuracy: 0.9204\n",
      "Epoch 9/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1152 - accuracy: 0.9612 - val_loss: 0.0689 - val_accuracy: 0.9750\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 10/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0986 - accuracy: 0.9715 - val_loss: 0.0895 - val_accuracy: 0.9669\n",
      "Epoch 11/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0959 - accuracy: 0.9692 - val_loss: 0.0486 - val_accuracy: 0.9838\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 12/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1235 - accuracy: 0.9659 - val_loss: 0.1939 - val_accuracy: 0.9393\n",
      "Epoch 13/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0912 - accuracy: 0.9690 - val_loss: 0.0850 - val_accuracy: 0.9625\n",
      "Epoch 14/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0926 - accuracy: 0.9685 - val_loss: 0.0841 - val_accuracy: 0.9706\n",
      "Epoch 15/50\n",
      "163/163 [==============================] - 18s 109ms/step - loss: 0.0942 - accuracy: 0.9691 - val_loss: 0.0554 - val_accuracy: 0.9821\n",
      "Epoch 16/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0800 - accuracy: 0.9768 - val_loss: 0.0680 - val_accuracy: 0.9821\n",
      "Epoch 17/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0782 - accuracy: 0.9779 - val_loss: 0.0491 - val_accuracy: 0.9852\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 18/50\n",
      "163/163 [==============================] - 18s 109ms/step - loss: 0.0736 - accuracy: 0.9785 - val_loss: 0.0527 - val_accuracy: 0.9804\n",
      "Epoch 19/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0788 - accuracy: 0.9765 - val_loss: 0.0414 - val_accuracy: 0.9848\n",
      "Epoch 20/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1056 - accuracy: 0.9673 - val_loss: 0.1061 - val_accuracy: 0.9679\n",
      "Epoch 21/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1189 - accuracy: 0.9671 - val_loss: 0.1534 - val_accuracy: 0.9528\n",
      "Epoch 22/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.2019 - accuracy: 0.9436 - val_loss: 0.0444 - val_accuracy: 0.9865\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 23/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0771 - accuracy: 0.9773 - val_loss: 0.0375 - val_accuracy: 0.9882\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 24/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0939 - accuracy: 0.9723 - val_loss: 0.0628 - val_accuracy: 0.9764\n",
      "Epoch 25/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1035 - accuracy: 0.9651 - val_loss: 0.1049 - val_accuracy: 0.9690\n",
      "Epoch 26/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1428 - accuracy: 0.9577 - val_loss: 0.0839 - val_accuracy: 0.9771\n",
      "Epoch 27/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1754 - accuracy: 0.9422 - val_loss: 0.0401 - val_accuracy: 0.9899\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 28/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1391 - accuracy: 0.9697 - val_loss: 0.1163 - val_accuracy: 0.9544\n",
      "Epoch 29/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.2032 - accuracy: 0.9297 - val_loss: 0.0462 - val_accuracy: 0.9852\n",
      "Epoch 30/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0927 - accuracy: 0.9717 - val_loss: 0.1103 - val_accuracy: 0.9740\n",
      "Epoch 31/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1055 - accuracy: 0.9676 - val_loss: 0.0810 - val_accuracy: 0.9744\n",
      "Epoch 32/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1032 - accuracy: 0.9714 - val_loss: 0.0531 - val_accuracy: 0.9814\n",
      "Epoch 33/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0658 - accuracy: 0.9816 - val_loss: 0.0539 - val_accuracy: 0.9892\n",
      "Epoch 34/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0596 - accuracy: 0.9813 - val_loss: 0.0278 - val_accuracy: 0.9895\n",
      "Epoch 35/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0538 - accuracy: 0.9835 - val_loss: 0.0310 - val_accuracy: 0.9885\n",
      "Epoch 36/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0550 - accuracy: 0.9831 - val_loss: 0.0363 - val_accuracy: 0.9868\n",
      "Epoch 37/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0739 - accuracy: 0.9757 - val_loss: 0.0292 - val_accuracy: 0.9899\n",
      "Epoch 38/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0580 - accuracy: 0.9806 - val_loss: 0.0243 - val_accuracy: 0.9912\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 39/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0542 - accuracy: 0.9842 - val_loss: 0.0298 - val_accuracy: 0.9916\n",
      "INFO:tensorflow:Assets written to: model/assets\n",
      "Epoch 40/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0587 - accuracy: 0.9801 - val_loss: 0.0433 - val_accuracy: 0.9845\n",
      "Epoch 41/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0762 - accuracy: 0.9792 - val_loss: 0.0416 - val_accuracy: 0.9828\n",
      "Epoch 42/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0566 - accuracy: 0.9825 - val_loss: 0.0229 - val_accuracy: 0.9916\n",
      "Epoch 43/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0633 - accuracy: 0.9811 - val_loss: 0.1155 - val_accuracy: 0.9585\n",
      "Epoch 44/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.3113 - accuracy: 0.8964 - val_loss: 0.3273 - val_accuracy: 0.9406\n",
      "Epoch 45/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.1848 - accuracy: 0.9552 - val_loss: 0.0461 - val_accuracy: 0.9835\n",
      "Epoch 46/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0806 - accuracy: 0.9726 - val_loss: 0.0294 - val_accuracy: 0.9899\n",
      "Epoch 47/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0590 - accuracy: 0.9807 - val_loss: 0.0483 - val_accuracy: 0.9821\n",
      "Epoch 48/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0728 - accuracy: 0.9793 - val_loss: 0.0421 - val_accuracy: 0.9865\n",
      "Epoch 49/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0590 - accuracy: 0.9848 - val_loss: 0.0491 - val_accuracy: 0.9875\n",
      "Epoch 50/50\n",
      "163/163 [==============================] - 18s 108ms/step - loss: 0.0715 - accuracy: 0.9805 - val_loss: 0.4326 - val_accuracy: 0.9494\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_dataset, epochs=50, validation_data=valid_dataset, \n",
    "                    callbacks=tf.keras.callbacks.ModelCheckpoint(filepath=MODEL_DIR, monitor='val_accuracy', mode='max', save_best_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_model = tf.keras.models.load_model(MODEL_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 22s 910ms/step - loss: 0.0278 - accuracy: 0.9906\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.02776455506682396, 0.9905533194541931]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_model.evaluate(test_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model achieves almost 99% on the test set, which is pretty nice!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_model.save(\"classify_applauses_model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}