train_TF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assessing quality of CO functionalized tips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aiARqQvAUCuO"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "from scipy import misc\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "from matplotlib import cm\n",
    "from utils import *\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload AFM data with bad and good samples of CO-tips\n",
    "Download AFM-data with CO-tips samples from here: https://www.dropbox.com/s/wqhgbvdlfb6snic/datasetNew6.tar.gz?dl=0. Extract the archive and set the path of the directory to the variable path_to_data below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_data = './databaseNew6'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p_jHZVAaSz7l"
   },
   "source": [
    "## PCA *analysis* "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 187
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 320272,
     "status": "ok",
     "timestamp": 1590744753824,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "mRR7_8b5UPxG",
    "outputId": "42407b50-7143-4075-fdbb-799069820db0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset founded\n",
      "trainY: 204\n",
      "valY: 69\n",
      "testY: 73\n",
      "dataset loaded succesfully\n"
     ]
    }
   ],
   "source": [
    "# Here we generate dataset for PCA and t-SNE analysis. All available images without augumentation are used (enable_rotations = False).\n",
    "\n",
    "save_dir         = './PCA3/'\n",
    "classes          = ['bads', 'goods']\n",
    "crop_size        = 16    # different size images would be transformed to size 32 x 32\n",
    "enable_rotations = False # no augumentation by rotations for PCA analysis\n",
    "enable_flips = False\n",
    "if os.path.exists(save_dir+'dataset_pca.npz'): \n",
    "    # if dataset was generated before, just load it\n",
    "    print ('dataset founded')\n",
    "    try:    \n",
    "        data= np.load(save_dir+'dataset_pca.npz')\n",
    "        trainX=data['trainX']\n",
    "        trainY=data['trainY']\n",
    "        valX=data['valX']\n",
    "        valY=data['valY']\n",
    "        testX=data['testX']\n",
    "        testY=data['testY']\n",
    "        print (f'trainY: {len(trainY)}')    \n",
    "        print (f'valY: {len(valY)}')    \n",
    "        print (f'testY: {len(testY)}')    \n",
    "        print ('dataset loaded succesfully')\n",
    "    except KeyError as e:\n",
    "        print('Could not find filename %s' % e)\n",
    "else:  \n",
    "    #no dataset founded, we generate new one and save it\n",
    "    if not os.path.exists(save_dir):\n",
    "        os.makedirs(save_dir)\n",
    "    # train data\n",
    "    print('gen training data') \n",
    "    trainX, trainY = generate_data_from_images(path_to_data, 'train', classes,crop_size, enable_rotations, enable_flips)\n",
    "    print (f'trainX: {trainX.shape}')\n",
    "    print (f'trainY: {trainY.shape}')\n",
    "\n",
    "    # validation data\n",
    "    print('gen validation data')  \n",
    "    valX, valY     = generate_data_from_images(path_to_data, 'valid', classes,crop_size, enable_rotations, enable_flips)\n",
    "    print (f'valX: {valX.shape}')\n",
    "    print (f'valY: {valY.shape}')\n",
    "\n",
    "    # test data\n",
    "    print('gen test data')  \n",
    "    testX, testY   = generate_data_from_images(path_to_data, 'test2', classes,crop_size, enable_rotations, enable_flips) \n",
    "    print (f'testX: {testX.shape}')\n",
    "    print (f'testY: {testY.shape}')\n",
    "    \n",
    "    # save whole PCA dataset\n",
    "    file_dataset = 'dataset_pca.npz'\n",
    "    np.savez(save_dir+ file_dataset , trainX=trainX,trainY=trainY, valX = valX, valY = valY, testX =testX, testY =testY  )\n",
    "    print (f'dataset saved succesfully to {save_dir+ file_dataset}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset statistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 83 121]\n",
      " [ 25  44]\n",
      " [ 36  37]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAAEYCAYAAADBOEomAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAfAUlEQVR4nO3de5xVdb3/8ddbRAHlYkKkoAKiJDcJB68cxDx5CfPyyKPiDUSd460kOynVOZp2Q3+mGWlGGZIKaahpevRoCiGaIuAgA2jhHUIFFBSBRPj8/lgL3A57YA/Mnj2z1/v5eMxj9l7Xz16befNdt+9SRGBmlgXblboAM7OG4sAzs8xw4JlZZjjwzCwzHHhmlhkOPDPLDAdeRkg6Q9Jjpa4DQFIPSVWSPpT0zXpc7hRJ523D/CGpe33VY42PA6+MSBoo6RlJKyS9J+lpSQMAIuKuiDiq1DWmLgcmR0TriPhFqYupK0ld0nDcvhzWkyUOvDIhqQ3wEDAG+BzQCbga+Fcp66rFXsDcUhdh2ePAKx/7AkTExIhYFxGrI+KxiHgRQNJwSdPS15dLWpnzs1bS7em4tpJuk7RY0iJJP5LULB3XXdJf0xbkUkl311aMpOMlzZW0PN3V3C8d/iRwBPDLdN375pl3uKRX013e1ySdkQ7/gaQ7c6bL1wLaW9J0SR9IekDS5zZT43fSz/lPSSNqjBsi6YV0OW9J+kHO6Knp7+XpZzhE0t6SnpS0LN02d0lql7O8K9Lt+aGklyUdmQ7fTtIoSa+k896TU3O+9RT8HVgeEeGfMvgB2gDLgPHAscAuNcYPB6blmW8P4J/Asen7+4FfAzsBnwemA/+ZjpsIfJ/kP8oWwMBaatkX+Aj4CtCcZBd2AbBDOn4KcF4t8+4EfAD0SN/vBvRKX/8AuDNn2i5AANvnLHcR0Dtdzr2509dYzzHAOznTTkiX1T0dPxjok37Wvum0J+Zbbzqse/p5dwQ6kITVz9NxPYC3gN1z5t87fX0p8CzQOZ3318DEzaynoO/AP/l/3MIrExHxATCQ5A/kN8ASSQ9K6ljbPJJaAn8CboqIR9JpvwqMjIiPIuJd4EbgtHSWtSS7o7tHxJqImFbLok8FHo6IxyNiLXA90BI4tMCPsx7oLallRCyOiLrs/t4REdUR8RHwP8ApG1qoNZwCjMuZ9ge5IyNiSkTMiYj1kbSSJwKH17bSiFiQft5/RcQS4Iac6deRhFlPSc0j4vWIeCUddwHw/YhYGBH/Sus4eTPH7Qr9DiwPB14ZiYj5ETE8IjqTtFx2B36+mVluA16OiGvT93uRtMgWp7uiy0laHJ9Px18OCJie7q6O2GSJid2BN3LqWk/SwulUwGf4iCQwL0jreFjSF7c0X463cl6/kX6e9rXUWHPajSQdJGmypCWSVqT15FvOhuk7SvpDutv6AXDnhukjYgEwkiTM3k2n2z2ddS/g/pztPZ8kIGv7j6rQ78DycOCVqYh4CbidJPg2IWkUya7nuTmD3yI5ydE+ItqlP20iole6zLcj4vyI2B34T+CWWi7j+CfJH/KGdYlk13lRgbX/X0R8hWR39iWSFisku8mtcib9Qp7Z98h5vSdJi2hpnukW55k21wTgQWCPiGgL3EoSNJC0omv6STq8T0S0Ac7MmZ6ImBARA0m2SwAb/pN5i+RwQrucnxYRsSjfeurwHVgeDrwyIemLkr4tqXP6fg9gKMnxoZrTHgt8EzgpIlZvGB4Ri4HHgJ9JapMeUN9b0uHpfP+xYfnA+yR/kOvzlHMPMETSkZKaA98mCdJnCvgcHSWdIGmndJ6VOeuoAgZJ2lNSW+C7eRZxpqSekloB1wCTImJdLTUOz5n2qhrjWwPvRcQaSQcCp+eMW5LW1K3G9CuBFZI6Ad/J+Uw9JH1Z0o7AGmB1zme6FfixpL3SaTtIOqG29dThO7A8HHjl40PgIOA5SR+RBF01SdjUdCrJgfX5+vRM7a3puLOBHYB5JH9Qk0haWgAD0uWvJGn9XBoRr9ZceES8TNLCGUPSuvoa8LWI+LiAz7EdcBlJK/E9kuNgF6bLfRy4G3gRmElyGU5Nd5C0bN8mOaif98LmiHiEZHf/SZITKk/WmOQi4BpJHwJXkgTkhnlXAT8Gnk53RQ8muQSoP7ACeBi4L2dZOwKjSbbF2ySHCDaE9U0k2/KxdF3PknyPta2noO/A8lOEOwA1s2xwC8/MMsOBZ2aZ4cAzs8xw4JlZZjTpXhjat28fXbp0KXUZZtbIzJw5c2lEdKg5vEkHXpcuXZgxY0apyzCzRkbSG/mGe5fWzDLDgWdmmeHAM7PMaNLH8Mwa2tq1a1m4cCFr1qwpdSkGtGjRgs6dO9O8efOCpnfgmdXBwoULad26NV26dCHpBMZKJSJYtmwZCxcupGvXrgXN411aszpYs2YNu+66q8OuEZDErrvuWqfWtgPPrI4cdo1HXb+LogWepN9JeldSdc6w/yfpJUkvSrq/xkNOvitpQfqAk6OLVZeZZVcxj+HdDvwS+H3OsMeB70bEJ5KuJekT7ApJPUmem9CLpOvtv0jat5aOG80ajYqK+l3elq6jf/311znuuOOorq7e/IRbMf/f//53Ro4cyT/+8Q9at25N9+7dGTNmDB07dmTatGlcdtllfPDBBwBcdtllVFZW5l3HT37yE773ve9tfH/ooYfyzDNb7Pu1QRQt8CJiqqQuNYY9lvP2WeDk9PUJwB/Sh5i8JmkBcCDwt2LVV0oVY+v5r2Qrzaj0XSqWWLNmDUOGDOGGG27ga1/7GgBTpkxhyZIlRASnn346f/rTn+jfvz9Lly7l6KOPplOnTgwZMmSTZdUMvMYSdlDaY3gjgEfS15347ANVFlLAA1/MsuiTTz7hjDPOYL/99uPkk09m1apVAFxzzTUMGDCA3r17U1lZueGxjsycOZP999+f/fffn5tvvjnvMidMmMAhhxyyMewABg8eTO/evbn55psZPnw4/fv3B6B9+/Zcd911jB49epPljBo1itWrV9OvXz/OOOMMAHbeeWcgCdBBgwYxZMgQevTowQUXXMD69etZt24dw4cPp3fv3vTp04cbb7yx/jZWDSUJPEnfBz4B7tqKeSslzZA0Y8mSJfVfnFkj9/LLL3PRRRcxf/582rRpwy233ALAJZdcwvPPP091dTWrV6/moYeSHvDPOeccxowZw+zZs2tdZnV1NQcccEDecXPnzt1kXEVFBXPnbvr0zNGjR9OyZUuqqqq4665N/7ynT5/OmDFjmDdvHq+88gr33XcfVVVVLFq0iOrqaubMmcM555xT8LaoqwYPPEnDgeOAM+LT/uUX8dknSHWmlidcRcTYiKiIiIoOHTbpDMGs7O2xxx4cdthhAJx55plMm5Y8mnby5MkcdNBB9OnThyeffJK5c+eyfPlyli9fzqBBgwA466yzSlY3wIEHHki3bt1o1qwZQ4cOZdq0aXTr1o1XX32Vb3zjGzz66KO0adOmaOtv0MCTdAzJczWPTx9QssGDwGmSdpTUFdiH5In3ZlZDzUsxJLFmzRouuugiJk2axJw5czj//PPrdH1ar169mDlzZt5xPXv23GTczJkz6dWrF+vWraNfv37069ePK6+8cqtq32WXXZg9ezaDBw/m1ltv5bzzziu47roq5mUpE0lOOvSQtFDSuSRnbVsDj0uq2vCkrPTJ8veQPCnrUeBin6E1y+/NN9/kb39LzudNmDCBgQMHbgy39u3bs3LlSiZNmgRAu3btaNeu3cZWYL7dTIDTTz+dZ555hocffnjjsKlTp1JdXc3FF1/M7bffTlVVFQDLli3jiiuu4PLLL6dZs2ZUVVVRVVXFNddcA0Dz5s1Zu3Zt3vVMnz6d1157jfXr13P33XczcOBAli5dyvr16/n617/Oj370I2bNmlUPWym/Yp6lHZpn8G2bmf7HJI+kM2syStEdY48ePbj55psZMWIEPXv25MILL6RVq1acf/759O7dmy984QsMGDBg4/Tjxo1jxIgRSOKoo47Ku8yWLVvy0EMPMXLkSEaOHEnz5s3p27cvN910Ex07duTOO+/k/PPP58MPPyQiGDly5GdOcOSqrKykb9++9O/ff5OAHTBgAJdccgkLFizgiCOO4KSTTtp43G79+uTxuj/96U/raUttqkk/prGioiKaYgegviyl6Zo/fz777bdfqctokqZMmcL111+/8WRKfcn3nUiaGRGb/KH51jIzywz3lmJmDWLw4MEMHjy4pDW4hWdmmeHAM7PMcOCZWWY48MwsM3zSwmwb1PclRqW6VCgr3UY58MysVuXWbZR3ac2amB/+8If06NGDgQMHMnToUK6//noAqqqqOPjgg+nbty8nnXQS77///maHZ7HbKAeeWRPy/PPPc++99zJ79mweeeQRcu80Ovvss7n22mt58cUX6dOnD1dfffVmh2ex2ygHnlkT8vTTT3PCCSfQokULWrduvbHltWLFCpYvX87hhx8OwLBhw5g6dWqtw7PabZQDz8xqVW7dRjnwzJqQww47jD//+c+sWbOGlStXbrwRv23btuyyyy489dRTANxxxx0cfvjhtQ7PardRPktrtg0a+jKSAQMGcPzxx9O3b186duxInz59aNu2LQDjx4/nggsuYNWqVXTr1o1x48ZtdngWu41y91Al4O6hmq7G0D3UypUr2XnnnVm1ahWDBg1i7NixG8+UNkXb2m1UXbqHcgvPrImprKxk3rx5rFmzhmHDhjXpsGtoDjyzJmbChAmlLqFeNWS3UT5pYVZHTfkwULmp63fhwDOrgxYtWrBs2TKHXiMQESxbtowWLVoUPI93ac3qoHPnzixcuBA/BL5xaNGiBZ07dy54egeeWR00b96crl27lroM20repTWzzHDgmVlmOPDMLDMceGaWGUULPEm/k/SupOqcYZ+T9Likf6S/d0mHS9IvJC2Q9KIkXzpuZvWumC2824FjagwbBTwREfsAT6TvAY4F9kl/KoFfFbEuM8uoogVeREwF3qsx+ARgfPp6PHBizvDfR+JZoJ2k3YpVm5llU0Mfw+sYEYvT128DHdPXnYC3cqZbmA7bhKRKSTMkzfDFn2ZWFyU7aRHJvTl1vj8nIsZGREVEVHTo0KEIlZlZuWrowHtnw65q+vvddPgiYI+c6Tqnw8zM6k1DB96DwLD09TDggZzhZ6dnaw8GVuTs+pqZ1Yui3UsraSIwGGgvaSFwFTAauEfSucAbwCnp5P8LfBVYAKwCtv15bGZmNRQt8CJiaC2jjswzbQAXF6sWMzPwnRZmliEOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZjjwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZjjwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZhTtubSNUUVFqStIVZa6ALNscgvPzDKjJIEn6VuS5kqqljRRUgtJXSU9J2mBpLsl7VCK2sysfDV44EnqBHwTqIiI3kAz4DTgWuDGiOgOvA+c29C1mVl5K9Uu7fZAS0nbA62AxcCXgUnp+PHAiSWqzczKVIMHXkQsAq4H3iQJuhXATGB5RHySTrYQ6NTQtZlZeSvFLu0uwAlAV2B3YCfgmDrMXylphqQZS5YsKVKVZlaOSrFL++/AaxGxJCLWAvcBhwHt0l1cgM7AonwzR8TYiKiIiIoOHTo0TMVmVhZKEXhvAgdLaiVJwJHAPGAycHI6zTDggRLUZmZlrBTH8J4jOTkxC5iT1jAWuAK4TNICYFfgtoauzczKW0nutIiIq4Cragx+FTiwBOWYWUb4TgszywwHnpllhgPPzDLDgWdmmeHAM7PMKCjwJF0nqY2k5pKekLRE0pnFLs7MrD4V2sI7KiI+AI4DXge6A98pVlFmZsVQaOBtuF5vCPDHiFhRpHrMzIqm0AuPH5L0ErAauFBSB2BN8coyM6t/BbXwImIUcChJp51rgVUkPZ6YmTUZhZ60aAVcBPwqHbQ70FgeiWNmVpBCj+GNAz4maeVB0nXTj4pSkZlZkRQaeHtHxHXAWoCIWAWoaFWZmRVBoYH3saSWQABI2hv4V9GqMjMrgkLP0l4FPArsIekukh6KhxerKDOzYigo8CLicUmzgINJdmUvjYilRa3MzKyebTbwJPWvMWhx+ntPSXtGxKzilGVmVv+21ML72WbGBcmzZM3MmoTNBl5EHNFQhZiZFVtBx/AktSC58HggScvuKeDWiPDtZWbWZBR6lvb3wIfAmPT96cAdwH8Uoygzs2IoNPB6R0TPnPeTJc0rRkFmZsVS6IXHsyQdvOGNpIOAGcUpycysOApt4R0APCPpzfT9nsDLkuYAERF9i1KdmVk9KjTwjilqFWZmDaDQOy3ekLQLsEfuPL7w2MyakkIvS/khyb2zr5B2IMA2XHgsqR3wW6B3upwRwMvA3UAXkudmnBIR72/N8s3M8il0l/YUki6iPq6n9d4EPBoRJ0vaAWgFfA94IiJGSxoFjAKuqKf1mZkVfJa2GmhXHyuU1BYYBNwGEBEfR8Ryki7jx6eTjQdOrI/1mZltUGgL76fAC5KqyekHLyKO34p1dgWWAOMk7Q/MBC4FOkbEhs4J3gY65ptZUiVQCbDnnntuxerNLKsKDbzxwLXAHGB9PayzP/CNiHhO0k0ku68bRURIinwzR8RYYCxARUVF3mnMzPIpNPBWRcQv6mmdC4GFEfFc+n4SSeC9I2m3iFgsaTfg3Xpan5kZUPgxvKck/VTSIZL6b/jZmhVGxNvAW5J6pIOOBOYBDwLD0mHDgAe2ZvlmZrUptIX3pfT3wTnDtqU/vG8Ad6VnaF8FziEJ33sknQu8QXJm2Mys3hR64XG99osXEVXkf67tkfW5HjOzXIW28JA0BOgFtNgwLCKuKUZRZmbFUNAxPEm3AqeS7IqKpB+8vYpYl5lZvSv0pMWhEXE28H5EXA0cAuxbvLLMzOpfoYG3Ov29StLuwCfAbsUpycysOAo9hvdQesP/dSR3RkBy87+ZWZOxpefSDgDeiogfpu93Jrnb4iXgxuKXZ2ZWf7a0S/tr4GMASYOA0emwFaS3d5mZNRVb2qVtFhHvpa9PBcZGxL3AvZKqiluamVn92lILr5mkDaF4JPBkzriCr+EzM2sMthRaE4G/SlpKcqb2KQBJ3Ul2a83MmozNBl5E/FjSEySXoDwWERu6Y9qO5CJkM7MmY4u7pRHxbJ5hfy9OOWZmxVPohcdmZk2eA8/MMsOBZ2aZ4cAzs8xw4JlZZjjwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsM9nlhZqBib76mfDW9G5YxSl2Cb4RaemWWGA8/MMsOBZ2aZUbLAk9RM0guSHkrfd5X0nKQFku6WtEOpajOz8lTKFt6lwPyc99cCN0ZEd+B94NySVGVmZaskgSepMzCE9Nm2kgR8GZiUTjIeOLEUtZlZ+SpVC+/nwOXA+vT9rsDyiPgkfb8Q6JRvRkmVkmZImrFkyZLiV2pmZaPBA0/SccC7ETFza+aPiLERURERFR06dKjn6sysnJXiwuPDgOMlfRVoAbQBbgLaSdo+beV1BhaVoDYzK2MN3sKLiO9GROeI6AKcBjwZEWcAk4GT08mGAQ80dG1mVt4a03V4VwCXSVpAckzvthLXY2ZlpqT30kbEFGBK+vpV4MBS1mNmn1Vu9yg3phaemVlRubcUs0aoonE0rKCy1AXUL7fwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZjjwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZjjwzCwzHHhmlhnu4t22ibsit6bELTwzywwHnpllhgPPzDLDgWdmmdHggSdpD0mTJc2TNFfSpenwz0l6XNI/0t+7NHRtZlbeStHC+wT4dkT0BA4GLpbUExgFPBER+wBPpO/NzOpNgwdeRCyOiFnp6w+B+UAn4ARgfDrZeODEhq7NzMpbSY/hSeoCfAl4DugYEYvTUW8DHWuZp1LSDEkzlixZ0iB1mll5KFngSdoZuBcYGREf5I6LiAAi33wRMTYiKiKiokOHDg1QqZmVi5IEnqTmJGF3V0Tclw5+R9Ju6fjdgHdLUZuZla9SnKUVcBswPyJuyBn1IDAsfT0MeKChazOz8laKe2kPA84C5kiqSod9DxgN3CPpXOAN4JQS1GZmZazBAy8ipgGqZfSRDVmLmWWL77Qws8xw4JlZZjjwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZjjwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZjjwzCwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZ2aZ4cAzs8xw4JlZZjS6wJN0jKSXJS2QNKrU9ZhZ+WhUgSepGXAzcCzQExgqqWdpqzKzctGoAg84EFgQEa9GxMfAH4ATSlyTmZUJRUSpa9hI0snAMRFxXvr+LOCgiLgkZ5pKoDJ92wN4ucEL3XbtgaWlLqLMeJsWR1PdrntFRIeaA7cvRSXbIiLGAmNLXce2kDQjIipKXUc58TYtjnLbro1tl3YRsEfO+87pMDOzbdbYAu95YB9JXSXtAJwGPFjimsysTDSqXdqI+ETSJcD/Ac2A30XE3BKXVQxNepe8kfI2LY6y2q6N6qSFmVkxNbZdWjOzonHgmVlmOPC2kaRdJVWlP29LWpTzfocClzFOUo9i11oqkiZLOrrGsJGSfrWZeVamv3eXNKmWaaZI2uwlE+l6WuW8/19J7er2CcpLffybTZczQtIXillrffMxvHok6QfAyoi4vsZwkWzr9SUprMTSi8UPiYhzcoY9C1weEVNrmWdlROy8heVOAf4rImZsZprXgYqIaIoXzxZdbf9mC5x3GnBJRFTVe2FF4hZekUjqLmmepLuAucBuksZKmiFprqQrc6adJqmfpO0lLZc0WtJsSX+T9PnSfYp6MwkYsqH1IKkLsDvwgqQnJM2SNEfSJrcRSuoiqTp93VLSHyTNl3Q/0DJnul/lbNur02HfTNczWdLkdNjrktqnry+TVJ3+jMxZ33xJv0mX9ZiklmSEpGGSpqetvVskbZf+u7wj/Y6qJX1T0qlAP+DuurYMS8mBV1xfBG6MiJ4RsQgYlV61vj/wlVo6RmgL/DUi9gf+BoxouHKLIyLeA6aTdAoByfWV9wCrgZMioj9wBPCztDVcmwuBVRGxH3AVcEDOuO+n27YvcLikvhHxC+CfwBERcUTugiQdAJwDHAQcDJwv6Uvp6H2AmyOiF7Ac+PpWfvQmRVJv4CTg0IjoR3LZ2mkk27l9RPSJiN7A7yPibqAKODUi+qX3vjd6DrzieqXG7tZQSbOAWcB+JD3C1LQ6Ih5JX88EuhS3xAYzkeSPh/T3REDATyS9CPwF6AR03MwyBgF3AkTEi8CLOeNOSbftC0Av8m/bXAOB+yPio4hYCdwH/Fs67rWc3bRy+g625N+BAcAMSVXA4cDewAKgh6RfpMdiV5Swxm3SqC48LkMfbXghaR/gUuDAiFgu6U6gRZ55cv+nXEf5fEcPADdK6g+0ioiZkoYDHYADImJterwt3zbZLEldgf8CBkTE+5Ju35rl5PhXzut15Ow6lzmRXOz/P5uMkPqStNAvJmnxVtacpilwC6/htAE+BD6QtBtw9BamLytpK2oy8DuS1h0ku+/vpmF3BLDXFhYzFTgdNu5+9U2HtyH5z2WFpI58uusMyTZvnWdZTwEnSmolaSeSXbmn6vzBystfSFrKG45x7ippT0kdSE66/RG4EuifTl/btm20yqX10BTMAuYBLwFvAE+XtpySmAjcz6e7tncBf5Y0B5hBsm0251fAOEnzgfkku5tExGxJL6Tzv8Vnt+1Y4FFJ/8w9jhcRs9KW4PR00G8j4oX0hEomRcSc9ITPXyRtB6wFLiBp5d6WHl8N4Ip0lnHAbyWtJtlzafTH8XxZipllhndpzSwzHHhmlhkOPDPLDAeemWWGA8/MMsOBZw1O0rr0/su56T3D304vg9jcPF0knV6EWj7Tm4qVNweelcLq9P7LXsBXSC4UvmoL83Qhvei4no0EHHgZ4cCzkoqId0luU7pEiS6Snkp7UJkl6dB00tHAv6Utw2/VNp2k3SRNTaerlvRv6fCj0t5nZkn6o6Sda/amIqmZpNvT+eZI+lYptokVjy88tganPH3dSVpO8mD1D4H1EbEmvf94YkRUSBpM0vfdcen0rWqZ7ttAi4j4saRmJK23HUk6Bzg2Ij6SdAWwY0Rco5z+8tIeVEZHxFfSdbSLiOUNsEmsgfjWMmtsmgO/lNSP5Jamfes43fPA7yQ1B/4UEVWSDifpPeXptPepHUi63qrpVaCbpDHAw8Bj9fSZrJFw4FnJSepGElrvkhzLe4ekz8DtgDW1zPatfNNFxFRJg4AhwO2SbgDeBx6PiKGbqyPtaWV/ko4dLgBOoQz6I7RP+RielVTaE8etwC8jOb7SFlicdod/FsnziWHTnjnyTidpL+CdiPgN8FuSnj2eBQ6T1D2dZidJ+9ZcbtpLyHYRcS/w33zaK4iVCbfwrBRaph1MNgc+Ae4AbkjH3QLcK+ls4FE+7VPwRWCdpNnA7ZuZbjDwHUlrgZXA2RGxJO17b6KkHdPp/hv4Ozm9qZCcsR2Xc4nMd+v7g1tp+aSFmWWGd2nNLDMceGaWGQ48M8sMB56ZZYYDz8wyw4FnZpnhwDOzzPj/Ril6Q9LI+CEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 324x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot dataset sizes\n",
    "import pandas as pd\n",
    "count_tips= []\n",
    "for tip_quality in [trainY, valY,testY]:\n",
    "    count_tips.append([sum(tip_quality == 0),sum(tip_quality == 1)])\n",
    "count_tips = np.array(count_tips, dtype=np.int32)\n",
    "print(count_tips)\n",
    "\n",
    "n_sets = 3\n",
    "# create plot\n",
    "fig = plt.figure(figsize=(4.5,4))\n",
    "index = np.arange(n_sets)\n",
    "bar_width = 0.4\n",
    "opacity = 0.8\n",
    "rects1 = plt.bar(index, count_tips[:,0], bar_width, alpha=opacity, color='b', label='bad CO-tips')\n",
    "rects2 = plt.bar(index + bar_width, count_tips[:,1], bar_width,alpha=opacity, color='g', label='good CO-tips')\n",
    "\n",
    "plt.ylabel('Samples')\n",
    "plt.xlabel('Datasets')\n",
    "plt.title('Sizes of sub datasets')\n",
    "plt.xticks(index + bar_width/2, ('Train', 'Validation', 'Test'))\n",
    "plt.legend()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('dataset_statistic.png', bbox_inches='tight', dpi=200)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_set_plots(path_to_data, set_name, outdir, classes,crop_size, enable_rotations, enable_flips):\n",
    "\n",
    "    dataX = []\n",
    "    dataY = []\n",
    "    file_names = []\n",
    "    z = len(classes)\n",
    "    if (enable_rotations):\n",
    "        amount_rot = 4\n",
    "    else:\n",
    "        amount_rot = 1\n",
    "\n",
    "    for class_ind, cls in enumerate(classes):\n",
    "\n",
    "        images = sorted(glob.glob(path_to_data+'/'+set_name+'/'+cls + '/' +'*.png'))\n",
    "        for image_path in images:\n",
    "            _, file_name = os.path.split(image_path)\n",
    "            file_name = file_name[:-4]\n",
    "            #image = np.array((Image.open(image_path).resize((crop_size,crop_size), Image.ANTIALIAS))).astype(np.float32)\n",
    "            image = np.array(Image.open(image_path)).astype(np.float32)\n",
    "            #print (image.shape) \n",
    "            for rotations in range(amount_rot): \n",
    "                    rot_image = np.rot90(image,rotations, axes=(-2,-1))             \n",
    "                    dataX.append(rot_image)\n",
    "                    dataY.append(class_ind)\n",
    "                    file_names.append(file_name)\n",
    "                    if (enable_flips):\n",
    "                        dataX.append(np.flipud(rot_image))\n",
    "                        dataY.append(class_ind)\n",
    "                        file_names.append(file_name)\n",
    "                        \n",
    "                        #dataX.append(np.fliplr(rot_image))\n",
    "                        #dataY.append(class_ind)\n",
    "                        #file_names.append(file_name)\n",
    "           \n",
    "        \n",
    "        \n",
    "    cmap=cm.gray\n",
    "    if not os.path.exists(outdir):\n",
    "        os.makedirs(outdir)\n",
    "\n",
    "    img_ind = 0\n",
    "     \n",
    "    cols = 10\n",
    "    rows = -(-len(file_names) // cols) \n",
    "    #rows = 4\n",
    "    #permut = np.array( range(X.shape[0]) )\n",
    "    #np.random.shuffle( permut )\n",
    "    txt_colors = ['red','green']\n",
    "    fig = plt.figure(figsize=(3*cols,3*rows))\n",
    "    permut = np.random.permutation(len(file_names))\n",
    "    for i in range(len(file_names)):\n",
    "    #for i in range(40):\n",
    "\n",
    "        sp = fig.add_subplot(rows,cols,img_ind+1)#, origin=\"lower\"\n",
    "        sp.imshow(dataX[img_ind], cmap = cmap)   \n",
    "        sp.axis('Off')\n",
    "        sp.set_title(f'{classes[dataY[img_ind]]}: {file_names[img_ind]}', color =txt_colors[dataY[img_ind]]) \n",
    "\n",
    "        if (img_ind > len(dataX)-2):\n",
    "            break\n",
    "        else:\n",
    "            img_ind+=1\n",
    "    save_name = f'{outdir}database_{set_name}_big.png'\n",
    "    plt.savefig(save_name, bbox_inches=\"tight\")\n",
    "    plt.close()\n",
    "    print('Input image saved to '+save_name)\n",
    "        \n",
    "            \n",
    " \n",
    "        \n",
    "enable_rotations = False\n",
    "enable_flips = False\n",
    "outdir = save_dir\n",
    "classes          = ['bads', 'goods']\n",
    "make_set_plots(path_to_data, 'train',outdir, classes,crop_size, enable_rotations, enable_flips)\n",
    "make_set_plots(path_to_data, 'valid',outdir, classes,crop_size, enable_rotations, enable_flips)\n",
    "make_set_plots(path_to_data, 'test2',outdir, classes,crop_size, enable_rotations, enable_flips)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_class_sets_plots(path_to_data, set_names, classes,class_ind, outdir,crop_size, enable_rotations, enable_flips):\n",
    "\n",
    "    dataX = []\n",
    "    dataY = []\n",
    "    file_names = []\n",
    "    z = len(classes)\n",
    "    if (enable_rotations):\n",
    "        amount_rot = 4\n",
    "    else:\n",
    "        amount_rot = 1\n",
    "    for set_name in set_names:\n",
    "        cls = classes[class_ind]\n",
    "        print (f'set_name = {set_name}')\n",
    "        print (f'cls = {cls}')\n",
    "        images = sorted(glob.glob(path_to_data+'/'+set_name+'/'+cls + '/' +'*.png'))\n",
    "        for image_path in images:\n",
    "            _, file_name = os.path.split(image_path)\n",
    "            file_name = file_name[:-4]\n",
    "            #image = np.array((Image.open(image_path).resize((crop_size,crop_size), Image.ANTIALIAS))).astype(np.float32)\n",
    "            image = np.array(Image.open(image_path)).astype(np.float32)\n",
    "            for rotations in range(amount_rot): \n",
    "                    rot_image = np.rot90(image,rotations, axes=(-2,-1))             \n",
    "                    dataX.append(rot_image)\n",
    "                    dataY.append(class_ind)\n",
    "                    file_names.append(file_name)\n",
    "                    if (enable_flips):\n",
    "                        dataX.append(np.flipud(rot_image))\n",
    "                        dataY.append(class_ind)\n",
    "                        file_names.append(file_name)\n",
    "\n",
    "        \n",
    "    cmap=cm.gray\n",
    "    if not os.path.exists(outdir):\n",
    "        os.makedirs(outdir)\n",
    "\n",
    "    img_ind = 0\n",
    "     \n",
    "    cols = 10\n",
    "    #rows = -(-len(file_names) // cols) \n",
    "    rows = 4\n",
    "    #permut = np.array( range(X.shape[0]) )\n",
    "    #np.random.shuffle( permut )\n",
    "    txt_colors = ['red','green']\n",
    "    fig = plt.figure(figsize=(3*cols,3*rows))\n",
    "    permut = np.random.permutation(len(file_names))\n",
    "    print (f'len files: {len(file_names)}')\n",
    "    #for i in range(len(file_names)):\n",
    "    for i in range(40):\n",
    "\n",
    "        sp = fig.add_subplot(rows,cols,img_ind+1)#, origin=\"lower\"\n",
    "        sp.imshow(dataX[permut[img_ind]], cmap = cmap)   \n",
    "        sp.axis('Off')\n",
    "        #sp.set_title(f'{classes[dataY[img_ind]]}: {file_names[img_ind]}', color =txt_colors[dataY[img_ind]]) \n",
    "\n",
    "        if (img_ind > len(dataX)-2):\n",
    "            break\n",
    "        else:\n",
    "            img_ind+=1\n",
    "    save_name = f'{outdir}database_{cls}_big_permut.png'\n",
    "    plt.savefig(save_name, bbox_inches=\"tight\")\n",
    "    plt.close()\n",
    "    print('Input image saved to '+save_name)\n",
    "    \n",
    "    \n",
    "        \n",
    "enable_rotations = False\n",
    "enable_flips = False\n",
    "outdir = save_dir\n",
    "classes          = ['bads', 'goods']\n",
    "sets = ['train','valid','test2']\n",
    "make_class_sets_plots(path_to_data, sets, classes,0, outdir,crop_size, enable_rotations, enable_flips)\n",
    "make_class_sets_plots(path_to_data, sets, classes,1, outdir, crop_size, enable_rotations, enable_flips)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 305261,
     "status": "ok",
     "timestamp": 1590744753825,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "YICSHguuS2O0",
    "outputId": "823fad41-b65b-48f1-8356-813be33a7951"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "save_dir = save_dir\n",
    "# Do Principal component analysis\n",
    "pca_data_X = trainX\n",
    "pca_data_X = pca_data_X.reshape((pca_data_X.shape[0],-1)) # reszie each image 1x32x32 --> 1x1024\n",
    "\n",
    "#scaler = MinMaxScaler()\n",
    "#pca_data_X = scaler.fit_transform(pca_data_X)\n",
    "print('Augmented data shape:', pca_data_X.shape)\n",
    "\n",
    "pca = PCA()\n",
    "T = pca.fit_transform(pca_data_X)\n",
    "print(f'explained_variance_ratio: {pca.explained_variance_ratio_[:5]}')\n",
    "print(f'explained_variance_ratio sum: {pca.explained_variance_ratio_[:5].cumsum()}') \n",
    "print(f'X_reduced: {T.shape}')\n",
    "print(f'trainY: {trainY.shape}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 314
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3007,
     "status": "ok",
     "timestamp": 1590635068983,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "t6j1mhtSStvY",
    "outputId": "00aabc88-162f-4b31-aaf9-a9a5ff728065"
   },
   "outputs": [],
   "source": [
    "# Plot explained variance as a function of number of components\n",
    "\n",
    "print(pca.explained_variance_ratio_[:10])\n",
    "plt.figure()\n",
    "n_comp = 20\n",
    "plt.plot(range(1,n_comp+1), pca.explained_variance_ratio_[:n_comp], 'x')\n",
    "plt.plot(range(1,n_comp+1),np.cumsum(pca.explained_variance_ratio_[:n_comp]))\n",
    "plt.xlabel('n_components')\n",
    "plt.legend(['% of explained variance', 'cumulative explained variance'])\n",
    "plt.savefig(save_dir + 'CO_tips_explained_variance.png', bbox_inches='tight', dpi=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 290
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2262,
     "status": "ok",
     "timestamp": 1590635125520,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "tSyrIQ1M1LoW",
    "outputId": "6538e94a-6b42-40ac-a7bd-d73ebb97cd55"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "classes          = ['bads', 'goods']\n",
    "colours = ListedColormap(['r','g'])\n",
    "\n",
    "# Visualize with two principle components\n",
    "fig = plt.figure(figsize=(9,4))\n",
    "ax = fig.add_subplot(121)\n",
    "scatter = plt.scatter(T[:,0], T[:,1], c=trainY.ravel(), cmap=colours )\n",
    "plt.xlim(T[:,0].min(), T[:,0].max())\n",
    "plt.ylim(T[:,1].min(), T[:,1].max())\n",
    "plt.xlabel('principal component 1')\n",
    "plt.ylabel('principal component 2')\n",
    "plt.legend(handles=scatter.legend_elements()[0], labels=classes)\n",
    "ax.ticklabel_format(axis='both', style='sci', scilimits=(0,0))\n",
    "\n",
    "# Visualize with three principle components\n",
    "ax = fig.add_subplot(122, projection='3d')\n",
    "scatter = ax.scatter(T[:,0], T[:,1], T[:,2], c=trainY.ravel(), cmap=colours)\n",
    "ax.set_xlabel('component 1')\n",
    "ax.set_ylabel('component 2')\n",
    "ax.set_zlabel('component 3')\n",
    "ax.ticklabel_format(axis='both', style='sci', scilimits=(0,0))\n",
    "plt.legend(handles=scatter.legend_elements()[0], labels=classes)\n",
    "plt.show()\n",
    "save_name = f'CO_tips_pca_no_rotations_cropsize{crop_size}.png'\n",
    "fig.savefig(save_dir+save_name, bbox_inches='tight', dpi=200)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "u1ttW9Xmf1Pb"
   },
   "source": [
    "## t-SNE  [t-distributed stochastic neighbor embedding] \n",
    "Visualizing data using t-SNE \n",
    "L.v.d. Maaten, G. Hinton, Journal of Machine Learning Research, Vol 9(Nov), pp. 2579—2605. 2008.\n",
    "https://distill.pub/2016/misread-tsne/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 399
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4459,
     "status": "ok",
     "timestamp": 1590637440748,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "Td6HcCr1eeqG",
    "outputId": "63a6ea20-1887-4acf-aa18-7ebd1ee58c5f"
   },
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "\n",
    "# t-SNE 2 components analysis\n",
    "tsne = TSNE(n_components=2, perplexity=30, learning_rate=200)\n",
    "E2 = tsne.fit_transform(pca_data_X)\n",
    "\n",
    "# t-SNE 3 components\n",
    "tsne = TSNE(n_components=3, perplexity=30, learning_rate=200)\n",
    "E3 = tsne.fit_transform(T[:,:50])\n",
    " \n",
    "# Visualize t-sne embedding: 2 components   \n",
    "fig = plt.figure(figsize=(9,4))\n",
    "ax = fig.add_subplot(121)    \n",
    "colours2 = ListedColormap(['b','m'])\n",
    "plt.scatter(E2[:,0], E2[:,1], c=trainY.ravel(), cmap=colours2)\n",
    "#plt.legend(handles=scatter.legend_elements()[0], labels=classes)\n",
    "\n",
    "plt.xlabel('t-sne component 1')\n",
    "plt.ylabel('t-sne component 2')\n",
    "ax.ticklabel_format(axis='both', style='sci', scilimits=(0,0))\n",
    "\n",
    "# Visualize t-sne embedding: 3 components   \n",
    "ax = fig.add_subplot(122, projection='3d')\n",
    "ax.scatter(E3[:,0], E3[:,1], E3[:,2], c=trainY.ravel(), cmap=colours2)\n",
    "#plt.legend(handles=scatter.legend_elements()[0], labels=classes)\n",
    "\n",
    "ax.set_xlabel('t-sne component 1')\n",
    "ax.set_ylabel('t-sne component 2')\n",
    "ax.set_zlabel('t-sne component 3')\n",
    "ax.ticklabel_format(axis='both', style='sci', scilimits=(0,0))\n",
    "\n",
    "plt.savefig(save_dir+'CO_tips_t-SNE_no_rotations_cropsize{crop_size}.png', bbox_inches='tight', dpi=200)\n",
    "plt.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 348
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8851,
     "status": "ok",
     "timestamp": 1590637675515,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "Y9byFuYiR8Ld",
    "outputId": "2c2d709b-b7af-4564-cbdc-e0f8a5e2fdb3"
   },
   "outputs": [],
   "source": [
    "# Let's vary parameters for t-SNE \n",
    "# here perplexities parameter varied. “perplexity,” says (loosely) \n",
    "# how to balance attention between local and global aspects of your data. \n",
    "# “The performance of SNE is fairly robust to changes in the perplexity, and typical values are between 5 and 50.”\n",
    "perplexities = [2 , 5, 10 , 30, 100, 500]   #range 2 to N\n",
    "\n",
    "E2_perpl = []\n",
    "for perplexity in perplexities:\n",
    "    print ('perplexity ', perplexity)\n",
    "    tsne = TSNE(n_components=2, perplexity=perplexity, learning_rate=200)\n",
    "    E2_perpl.append(tsne.fit_transform(pca_data_X))   \n",
    "\n",
    "\n",
    "plt.figure(figsize=(5*len(perplexities),4))\n",
    "for i, d in enumerate(E2_perpl):\n",
    "    plt.subplot(101+len(perplexities)*10+i)\n",
    "    colours2 = ListedColormap(['b','m'])\n",
    "    plt.scatter(d[:,0], d[:,1], c=trainY.ravel(), cmap=colours2)\n",
    "    plt.xlabel('t-sne component 1')\n",
    "    plt.ylabel('t-sne component 2')\n",
    "    ax.ticklabel_format(axis='both', style='sci', scilimits=(0,0))\n",
    "    #plt.legend(handles=scatter.legend_elements()[0], labels=classes)\n",
    "    plt.title('perplexity '+str(perplexities[i]))\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.savefig(save_dir+'CO_tips_t-SNE_2D_perplexity_var.png', dpi=200, bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 372
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5700,
     "status": "ok",
     "timestamp": 1590638457379,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "PT6bnyfrZOFJ",
    "outputId": "c491b26b-b9d7-4da7-b965-8bfa1a2378ae"
   },
   "outputs": [],
   "source": [
    "# here n_iterations is varied\n",
    "\n",
    "iterations   = [250, 300, 350, 500, 1000] \n",
    "\n",
    "E2 = []\n",
    "for iteration in iterations:\n",
    "    print ('n_iter ', iteration)\n",
    "    tsne = TSNE(n_components=2, perplexity=30, learning_rate=200, n_iter = iteration)\n",
    "    E2.append(tsne.fit_transform(pca_data_X))   \n",
    "\n",
    "plt.figure(figsize=(5*len(iterations),4))\n",
    "for i, d in enumerate(E2):\n",
    "    plt.subplot(101+len(iterations)*10+i)\n",
    "    colours2 = ListedColormap(['b','m'])\n",
    "    plt.scatter(d[:,0], d[:,1], c=trainY.ravel(), cmap=colours2)\n",
    "    plt.xlabel('t-sne component 1')\n",
    "    plt.ylabel('t-sne component 2')\n",
    "    ax.ticklabel_format(axis='both', style='sci', scilimits=(0,0))\n",
    "    #plt.legend(handles=scatter.legend_elements()[0], labels=classes)\n",
    "    plt.title('n_iter '+str(iterations[i]))\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.savefig(save_dir+'CO_tips_t-SNE_2D_n_iter_var.png', dpi=200, bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "P-fFkbtRquue"
   },
   "source": [
    "## CNN analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1743,
     "status": "ok",
     "timestamp": 1590649464543,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "B_7oaT08xyrA",
    "outputId": "7085b8c6-d361-415b-d496-ac131817c908"
   },
   "outputs": [],
   "source": [
    "\n",
    "import sys\n",
    "from sklearn.metrics import confusion_matrix, classification_report, roc_curve, auc\n",
    "from utils import *\n",
    "import os\n",
    "\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(2)\n",
    "np.random.seed(1)\n",
    "from keras import backend\n",
    "\n",
    "from keras.callbacks import ModelCheckpoint, CSVLogger\n",
    "from keras.utils     import plot_model\n",
    "\n",
    "assert len(backend.tensorflow_backend._get_available_gpus()) > 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1868,
     "status": "ok",
     "timestamp": 1590649518599,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "kxJ5gD0KqxJ_",
    "outputId": "93fb0ef8-c993-40a7-e67e-ce922628e1ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gen training data\n",
      "trainX: (1632, 16, 16)\n",
      "trainY: (1632,)\n",
      "gen validation data\n",
      "valX: (552, 16, 16)\n",
      "valY: (552,)\n",
      "gen test data\n",
      "testX: (73, 16, 16)\n",
      "testY: (73,)\n",
      "dataset cnn saved succesfully to ./CNN5_tf/dataset_cnn.npz\n",
      "Total amount of samples:  2257\n"
     ]
    }
   ],
   "source": [
    "# CNN training parameters\n",
    "path_to_data = './databaseNew6'\n",
    "descriptors      = ['class']\n",
    "classes          = ['bads', 'goods']\n",
    "batch_size       = 32\n",
    "pred_batches     = 10 \n",
    "verbose          = 1\n",
    "crop_size        = 16\n",
    "enable_rotations = True \n",
    "enable_flips     = True\n",
    "save_dir         = './CNN_tf/'\n",
    "\n",
    "model_dir         = save_dir\n",
    "pred_dir          = os.path.join(model_dir, 'predictions/')     # Where to save predictions\n",
    "checkpoint_dir    = os.path.join(model_dir, 'checkpoints/')     # Where to save model checkpoints\n",
    "log_path          = os.path.join(model_dir, 'training.log')     # Where to save loss history during training\n",
    "batch_dir         = model_dir+'batch_imgs/'\n",
    "if not os.path.exists(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "\n",
    "\n",
    "if os.path.exists(save_dir+'dataset_cnn.npz'):\n",
    "    print ('dataset cnn founded')\n",
    "    try:    \n",
    "        data= np.load(save_dir+'dataset_cnn.npz')\n",
    "        trainX=data['trainX']\n",
    "        trainY=data['trainY']\n",
    "        valX=data['valX']\n",
    "        valY=data['valY']\n",
    "        testX=data['testX']\n",
    "        testY=data['testY']\n",
    "        print (f'trainY: {len(trainY)}')    \n",
    "        print (f'valY: {len(valY)}')    \n",
    "        print (f'testY: {len(testY)}')    \n",
    "        print ('dataset loaded succesfully')\n",
    "    except KeyError as e:\n",
    "        print('Could not find filename %s' % e)\n",
    "else:\n",
    "\n",
    " \n",
    "    if not os.path.exists(save_dir):\n",
    "        os.makedirs(save_dir)\n",
    "\n",
    "    # train\n",
    "    print('gen training data') \n",
    "    trainX, trainY = generate_data_from_images(path_to_data, 'train', classes,crop_size, enable_rotations, enable_flips)\n",
    "    print (f'trainX: {trainX.shape}')\n",
    "    print (f'trainY: {trainY.shape}')\n",
    "\n",
    "    # validation\n",
    "    print('gen validation data')  \n",
    "    valX, valY     = generate_data_from_images(path_to_data, 'valid', classes,crop_size, enable_rotations, enable_flips)\n",
    "    print (f'valX: {valX.shape}')\n",
    "    print (f'valY: {valY.shape}')\n",
    "\n",
    "    # test  generate test data without augumentations\n",
    "    print('gen test data')  \n",
    "    testX, testY   = generate_data_from_images(path_to_data, 'test2', classes,crop_size, enable_rotations=False, enable_flips=False) \n",
    "    print (f'testX: {testX.shape}')\n",
    "    print (f'testY: {testY.shape}')\n",
    "    file_dataset = 'dataset_cnn.npz'\n",
    "    np.savez(save_dir+ file_dataset , trainX=trainX,trainY=trainY, valX = valX, valY = valY, testX =testX, testY =testY  )\n",
    "    print (f'dataset cnn saved succesfully to {save_dir+ file_dataset}')\n",
    "train_len = trainY.shape[0]//batch_size\n",
    "val_len = valY.shape[0]//batch_size\n",
    "test_len = testY.shape[0]//batch_size\n",
    "print ('Total amount of samples: ', trainY.shape[0] +valY.shape[0] + testY.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, AveragePooling2D, MaxPooling2D\n",
    "from keras.layers import Dropout,SpatialDropout2D, Flatten, Dense,LeakyReLU,Activation, ReLU\n",
    "from keras import backend as K\n",
    "\n",
    "\n",
    "def custom_loss():\n",
    "    # Create a loss function that adds the MSE loss to the mean of all squared activations of a specific layer\n",
    "    def loss(y_true, y_pred):\n",
    "        #return tf.nn.sigmoid_cross_entropy_with_logits(labels=y_true, logits=y_pred) \n",
    "        return K.mean(K.binary_crossentropy(y_true, y_pred, from_logits=True),axis = -1)\n",
    "    \n",
    "    # Return a function\n",
    "    return loss\n",
    "\n",
    "def conv_net_tf(lrelu_factor = 0.1):\n",
    "    # create model\n",
    "    def activation():\n",
    "        return LeakyReLU(alpha=lrelu_factor)\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, (3, 3), input_shape=input_shape, kernel_initializer = 'he_uniform'))\n",
    "    model.add(activation())\n",
    "    model.add(SpatialDropout2D(0.05))\n",
    "    model.add(Conv2D(4, (3, 3), kernel_initializer = 'he_uniform'))\n",
    "    model.add(activation())\n",
    "    model.add(SpatialDropout2D(0.05))\n",
    "    model.add(AveragePooling2D(pool_size=(2, 2)))\n",
    "\n",
    "    model.add(Conv2D(8, (3, 3), kernel_initializer = 'he_uniform'))\n",
    "    model.add(activation())\n",
    "    model.add(SpatialDropout2D(0.05))\n",
    "    model.add(Conv2D(8, (3, 3), kernel_initializer = 'he_uniform'))\n",
    "    model.add(activation())\n",
    "    model.add(SpatialDropout2D(0.05))\n",
    "\n",
    "\n",
    "\n",
    "    model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors\n",
    "    model.add(Dense(32))\n",
    "    model.add(activation())\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(1))\n",
    "    model.add(Activation('sigmoid'))\n",
    "\n",
    "    # Compile model\n",
    "    model.compile(loss = 'binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_shape = (1, 16, 16)\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 4, 14, 14)         40        \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)    (None, 4, 14, 14)         0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_1 (Spatial (None, 4, 14, 14)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 4, 12, 12)         148       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)    (None, 4, 12, 12)         0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_2 (Spatial (None, 4, 12, 12)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 4, 6, 6)           0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 8, 4, 4)           296       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)    (None, 8, 4, 4)           0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_3 (Spatial (None, 8, 4, 4)           0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 8, 2, 2)           584       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)    (None, 8, 2, 2)           0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_4 (Spatial (None, 8, 2, 2)           0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 33        \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 2,157\n",
      "Trainable params: 2,157\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "K.set_image_dim_ordering('th')\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    input_shape = (1,crop_size, crop_size)\n",
    "else:\n",
    "    input_shape = (crop_size, crop_size,1)\n",
    "print(f'input_shape = {input_shape}')\n",
    "model = conv_net_tf()\n",
    "model.summary()\n",
    "plot_model(model, to_file=os.path.join(model_dir, 'model.png'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "86bPrOEaVW5j"
   },
   "source": [
    "### Training and validation processes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 116448,
     "status": "ok",
     "timestamp": 1590649652524,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "S54mh59Xtpil",
    "outputId": "56c39431-252b-4d06-bec3-86d3eb1f53dc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizer weights loaded from ./CNN5_tf/optimizer_state.pkl\n",
      "\n",
      " === Starting training loop \n",
      "\n",
      "Using existing log loss at ./CNN5_tf/loss_log.csv\n",
      "Using existing log accu at ./CNN5_tf/accu_log.csv\n",
      "Model weights loaded from ./CNN5_tf/checkpoints/weights_29.h5\n",
      "\n",
      "Model saved to ./CNN5_tf/model.h5\n"
     ]
    }
   ],
   "source": [
    "epochs           = 29\n",
    "\n",
    "# Load optimizer state if available\n",
    "optimizer_path = os.path.join(model_dir, 'optimizer_state.pkl')\n",
    "load_optimizer_state(model, optimizer_path)\n",
    "\n",
    "# Create output folder\n",
    "if not os.path.exists(model_dir):\n",
    "    os.makedirs(model_dir)\n",
    "\n",
    "# Create a folder for model checkpoints\n",
    "if not os.path.exists(checkpoint_dir):\n",
    "    os.makedirs(checkpoint_dir)\n",
    "\n",
    "# Create a folder for model predictions\n",
    "if not os.path.exists(pred_dir):\n",
    "    os.makedirs(pred_dir)\n",
    "\n",
    "# Setup callbacks\n",
    "checkpointer = ModelCheckpoint(checkpoint_dir+'weights_{epoch:d}.h5', save_weights_only=True)\n",
    "logger = CSVLogger(log_path, append=True)\n",
    "\n",
    "\n",
    "\n",
    "print('\\n === Starting training loop \\n')\n",
    "start_time = time.time()\n",
    "# Initialize loss log\n",
    "loss_log = LossAccuLog(model_dir, descriptors=descriptors)\n",
    "\n",
    "# Resume previous epoch if exists\n",
    "init_epoch = 0\n",
    "model_file = None\n",
    "for i in range(1, epochs+1):\n",
    "    cp_file = os.path.join(checkpoint_dir, 'weights_%d.h5' % i)\n",
    "    if os.path.exists(cp_file):\n",
    "        init_epoch += 1\n",
    "        model_file = cp_file\n",
    "    else:\n",
    "        break\n",
    "if init_epoch > 0:\n",
    "    model.load_weights(model_file)\n",
    "    print('Model weights loaded from '+model_file)\n",
    "    \n",
    "\n",
    "\n",
    "for epoch in range(init_epoch, epochs):\n",
    "    train_loss = np.zeros(1)\n",
    "    val_loss = np.zeros(1)\n",
    "    train_accu = np.zeros(1)\n",
    "    val_accu = np.zeros(1)\n",
    "    epoch_start = time.time()\n",
    "\n",
    "    # Train\n",
    "    preds_train = np.zeros(train_len*batch_size)\n",
    "    for batch in range(train_len):\n",
    "        batch_indexes = np.arange(batch*batch_size,(batch+1)*batch_size)\n",
    "\n",
    "        X = np.expand_dims(trainX[batch_indexes], axis = 1)\n",
    "\n",
    "        Y = trainY[batch_indexes] \n",
    "        add_norm_CO(X)\n",
    "        # Train model\n",
    "        loss = model.train_on_batch(X, Y)\n",
    "\n",
    "        loss = np.array(loss)\n",
    "        loss = np.mean(loss)\n",
    "        outputs = model.predict_on_batch(X)\n",
    "        predicted = np.squeeze(np.array(outputs))\n",
    "        #if (batch < 2) and ((epoch+1) % 2 ==0):\n",
    "        #    make_batch_plots(X,Y,predicted,epoch+1, batch+1,'train',classes, outdir=model_dir)\n",
    "        #print (f'outputs = {outputs[:5]}')\n",
    "        predicted_tags = np.squeeze(np.round(outputs))\n",
    "        #print (f'predicted_tags = {predicted_tags[:5]}, predicted_tags.shape = {predicted_tags.shape}')\n",
    "        #print (f'Y = {Y[:5]}, Y.shape = {Y.shape}')\n",
    "        preds_train[batch_indexes] = np.round(predicted)\n",
    "        total = Y.shape[0]\n",
    "        correct = (predicted_tags == Y).sum()\n",
    "        #print (f'correct = {correct}, total = {total}')\n",
    "        accuracy = 100. * correct / total   \n",
    "\n",
    "        # Update mean loss\n",
    "        train_loss = train_loss + (loss - train_loss) / (batch+1)\n",
    "\n",
    "        # Update mean accuracy\n",
    "        # print('%f (Fit) Updating accuracy % time.time())\n",
    "        train_accu = train_accu + (accuracy - train_accu) / (batch+1)        \n",
    "\n",
    "\n",
    "        # Print information\n",
    "#                print('%f (Fit) Printing information' % time.time())\n",
    "        if (batch+1 == train_len):\n",
    "\n",
    "            eta = (time.time() - epoch_start) / (batch+1) * ((train_len+val_len) - (batch + 1))\n",
    "            msg = 'Epoch '+str(epoch+1)+', Batch '+str(batch+1)\n",
    "            for i in range(len(train_loss)):\n",
    "                msg +=  ' - Loss '+': %.4g'% train_loss[i]\n",
    "                msg +=  ' - Accuracy '+': %.4g'% train_accu[i]\n",
    "            msg += ' - ETA: %ds' % eta\n",
    "            print(msg)\n",
    "            print (f'loss item = {loss.item()}')\n",
    "            #print (f'outputs = {outputs}')\n",
    "    #print('Reset scheduler')\n",
    "    #scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, steps)                \n",
    "        # Task done on queue item\n",
    "#                print('%f (Fit) Task done' % time.time())\n",
    "\n",
    "\n",
    "\n",
    "    # Validate\n",
    "    preds_val = np.zeros(val_len*batch_size)\n",
    "    for batch in range(val_len):\n",
    "    #for batch in range(20): \n",
    "        batch_indexes = np.arange(batch*batch_size,(batch+1)*batch_size)\n",
    "\n",
    "        # Get new batch from queue when available\n",
    "        X = np.expand_dims(valX[batch_indexes], axis = 1)\n",
    "\n",
    "        Y = valY[batch_indexes]\n",
    "        #print (f'X.shape = {X.shape}')\n",
    "        #print (f'Y.shape = {Y.shape}')\n",
    "           \n",
    "        add_norm_CO(X)\n",
    "        \n",
    "        loss = model.test_on_batch(X, Y)\n",
    "        loss = np.array(loss)\n",
    "        loss = np.mean(loss)\n",
    "        outputs = model.predict_on_batch(X)\n",
    "        predicted = np.squeeze(np.array(outputs))\n",
    "        #if (batch < 2) and ((epoch +1)% 2 ==0):\n",
    "        #    make_batch_plots(X,Y,predicted,epoch+1, batch+1,'valid', classes, outdir=model_dir)\n",
    "\n",
    "        predicted_tags = np.squeeze(np.round(outputs))\n",
    "        preds_val[batch_indexes] = np.round(predicted)\n",
    "        #print (f'predicted = {predicted}')                \n",
    "        total = Y.shape[0]\n",
    "        correct = (predicted_tags == Y).sum().item()    \n",
    "        accuracy = 100. * correct / total  \n",
    "        \n",
    "        # Update mean loss\n",
    "        val_loss += (loss.item() - val_loss) / (batch+1)\n",
    "        val_accu += (accuracy - val_accu) / (batch+1)\n",
    "        # Print information\n",
    "        if verbose > 0:\n",
    "            eta = (time.time() - epoch_start) / (train_len+batch+1) * (val_len - (batch + 1))\n",
    "            #print('ETA: %ds' % eta)\n",
    "\n",
    "\n",
    "    if verbose > 0:\n",
    "        msg = 'Validation'\n",
    "        for i in range(len(val_loss)):\n",
    "            msg +=  ' - Loss '+': %.4g'%val_loss[i]\n",
    "            msg +=  ' - Accuracy '+': %.4g'%val_accu[i]\n",
    "        print(msg)\n",
    "\n",
    "    if verbose > 0:\n",
    "        time_diff = time.time() - epoch_start\n",
    "        print('Epoch time: %ds' % time_diff + ' - Step time: %.4gs' % (time_diff/(train_len+val_len)))\n",
    "\n",
    "    # Write loss history into log\n",
    "    loss_log.add_loss(np.append(train_loss,val_loss))\n",
    "    loss_log.add_accu(np.append(train_accu,val_accu))\n",
    "\n",
    "    # Save model checkpoint\n",
    "    model.save_weights(os.path.join(checkpoint_dir, f'weights_{epoch}.h5'))\n",
    "    # Save optimizer weights\n",
    "    save_optimizer_state(model, optimizer_path)\n",
    "    # Plot loss history\n",
    "    loss_log.plot_history_loss()\n",
    "    loss_log.plot_history_accu()\n",
    "\n",
    "    # Plot confusion matrices\n",
    "    conf_mat_train = confusion_matrix(trainY[:train_len*batch_size], preds_train)\n",
    "    conf_mat_val = confusion_matrix(valY[:val_len*batch_size], preds_val)\n",
    "    tick_labels = classes\n",
    "\n",
    "    # Training set\n",
    "    fig = plt.figure(figsize=(9.5,3.5))\n",
    "    ax = fig.add_subplot(121)\n",
    "    plot_confusion_matrix(ax, conf_mat_train, tick_labels)\n",
    "    ax.set_title('Training set')\n",
    "\n",
    "    # Validation set\n",
    "    ax = fig.add_subplot(122)\n",
    "    plot_confusion_matrix(ax, conf_mat_val, tick_labels)\n",
    "    ax.set_title('Validation set')\n",
    "    plt.savefig(model_dir+'predictions/'+'epoch'+str(epoch+1)+'_conf_mat_train_val.png', bbox_inches='tight', dpi=200)\n",
    "    print ('confustion matrix for validation plotted')\n",
    "    plt.close()\n",
    "\n",
    "    # shuffle training and validatation data\n",
    "    permut = np.random.permutation(trainX.shape[0])\n",
    "    trainX, trainY = trainX[permut], trainY[permut]\n",
    "\n",
    "\n",
    "# Save final model\n",
    "savepath = os.path.join(model_dir, 'model.h5')\n",
    "model.save(savepath )\n",
    "print(f'\\nModel saved to {savepath}')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fHpp1RX8VdNx"
   },
   "source": [
    "### Evaluation on a test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rm -rf CNN5_tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_path = :./CNN5_tf/model.h5\n",
      "\n",
      "Model loaded from ./CNN5_tf/model.h5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 4, 14, 14)         40        \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)    (None, 4, 14, 14)         0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_1 (Spatial (None, 4, 14, 14)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 4, 12, 12)         148       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)    (None, 4, 12, 12)         0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_2 (Spatial (None, 4, 12, 12)         0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 4, 6, 6)           0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 8, 4, 4)           296       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)    (None, 8, 4, 4)           0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_3 (Spatial (None, 8, 4, 4)           0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 8, 2, 2)           584       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_4 (LeakyReLU)    (None, 8, 2, 2)           0         \n",
      "_________________________________________________________________\n",
      "spatial_dropout2d_4 (Spatial (None, 8, 2, 2)           0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_5 (LeakyReLU)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 33        \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 2,157\n",
      "Trainable params: 2,157\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "test batch 0\n",
      "ETA: 0s\n",
      "test batch 1\n",
      "Statistical information saved to ./CNN5_tf/metrics.csv\n",
      "precision-recall curve (bads) plot saved to ./CNN5_tf/prec-rec_bads.png\n",
      "precision-recall curve (goods) plot saved to ./CNN5_tf/prec-rec_goods.png\n",
      "ETA: 0s\n",
      "Accuracy on test set: 90.625\n",
      "Test set - Loss class: 0.6167\n",
      "conf_mat_test = [[30  6]\n",
      " [ 0 28]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.83      0.91        36\n",
      "           1       0.82      1.00      0.90        28\n",
      "\n",
      "    accuracy                           0.91        64\n",
      "   macro avg       0.91      0.92      0.91        64\n",
      "weighted avg       0.92      0.91      0.91        64\n",
      "\n",
      "\n",
      " ======= Predict \n",
      "test_len =  2\n",
      "Input image saved to ./CNN5_tf/predictions/epoch29_batch1_test.png\n",
      "Input image saved to ./CNN5_tf/predictions/epoch29_batch2_test.png\n",
      "\n",
      "Predictions finished in 3.7034528255462646s\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "model_path = save_dir + 'model.h5'\n",
    "print(f'model_path = :{model_path}')\n",
    "# Load model\n",
    "model = load_model(model_path)\n",
    "print(f'\\nModel loaded from {model_path}')\n",
    "# summarize model.\n",
    "model.summary()\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "eval_loss = np.zeros(len(descriptors))\n",
    "eval_start = time.time()\n",
    "total = 0\n",
    "correct = 0    \n",
    "\n",
    "n_classes = len(classes)\n",
    "accu_metrics = accuMetrics(n_classes=n_classes)\n",
    "\n",
    "#for batch in range(test_len):\n",
    "preds_test = np.zeros(test_len*batch_size)\n",
    "for batch in range(test_len):       \n",
    "    print (f'test batch {batch}')\n",
    "    batch_indexes = np.arange(batch*batch_size,(batch+1)*batch_size)\n",
    "    \n",
    "    # Get new batch from queue when available\n",
    "    X = np.expand_dims(testX[batch_indexes], axis = 1)\n",
    "\n",
    "    Y = testY[batch_indexes]\n",
    "    \n",
    "    #X = X [:,2,:,:]\n",
    "    #X = np.expand_dims(X, axis=1)             \n",
    "    #for i in range(X.shape[0]):\n",
    "    #    X[i,2,:,:] = np.clip(X[i,2,:,:], None, adhesive_treshold*np.max(X[i,2,:,:]))\n",
    "    add_norm_CO(X)\n",
    "    #X = X * weight \n",
    "\n",
    "\n",
    "    # Test on batch\n",
    "    loss = model.test_on_batch(X, Y)\n",
    "    loss = np.array(loss)\n",
    "    loss = np.mean(loss)\n",
    "    outputs = model.predict_on_batch(X)\n",
    "    predicted = np.squeeze(np.array(outputs))\n",
    "\n",
    "\n",
    "    eval_loss += (loss.mean() - eval_loss) / (batch+1) \n",
    "    #print(f'predicted = {predicted}')\n",
    "    #print(f'true = {Y.cpu().numpy()}')\n",
    "    accu_metrics.add_preds(predicted , Y, loss)\n",
    "    if batch == test_len - 1:\n",
    "        accu_metrics.plot(outdir=save_dir)\n",
    "    predicted_tags = np.squeeze(np.round(outputs))\n",
    "    preds_test[batch_indexes] = np.round(predicted)\n",
    "    total   += Y.shape[0]\n",
    "    correct += (predicted_tags == Y).sum()\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "    if verbose > 0:\n",
    "        eta = (time.time() - eval_start) / (batch+1) * (test_len - (batch + 1))\n",
    "        print('ETA: %ds' % eta)\n",
    "print(f'Accuracy on test set: {100 * correct / total}')\n",
    "msg = 'Test set'\n",
    "for i in range(len(eval_loss)):\n",
    "    msg += ' - Loss '+descriptors[i]+': %.4g'%eval_loss[i]\n",
    "print(msg)\n",
    "\n",
    "with open(model_dir+'test_loss.txt','w') as f:\n",
    "    for descriptor in descriptors:\n",
    "        f.write(descriptor+';')\n",
    "    for loss in eval_loss:\n",
    "        f.write(str(loss)+';')\n",
    "\n",
    "# Plot confusion matrices\n",
    "conf_mat_test = confusion_matrix(testY[:test_len*batch_size], preds_test)\n",
    "print (f'conf_mat_test = {conf_mat_test}')         \n",
    "tick_labels = classes\n",
    "\n",
    "fig = plt.figure(figsize=(4.7,3.5))\n",
    "ax = fig.add_subplot(111)\n",
    "plot_confusion_matrix(ax, conf_mat_test, tick_labels)\n",
    "ax.set_title('Test set')\n",
    "plt.savefig(model_dir+'predictions/'+'epoch'+str(epochs)+'_conf_mat_test_set.png', bbox_inches='tight', dpi=200)\n",
    "plt.close() \n",
    "\n",
    "\n",
    "print(classification_report(testY[:test_len*batch_size], preds_test))\n",
    "\n",
    " \n",
    "print('\\n ======= Predict ')\n",
    "if len(descriptors) > 1:\n",
    "    descriptors = descriptors[1:]\n",
    "\n",
    "counter = 0\n",
    "print ('test_len = ', test_len)\n",
    "\n",
    "for i in range(min(pred_batches, test_len)):\n",
    "    batch_indexes = np.arange(i*batch_size,(i+1)*batch_size)\n",
    "    X = np.expand_dims(testX[batch_indexes], axis = 1)\n",
    "    true = testY[batch_indexes]\n",
    "\n",
    "    add_norm_CO(X)\n",
    "    #X = X * weight \n",
    "    # Calculate loss on batch\n",
    "    \n",
    "    outputs = model.predict_on_batch(X)\n",
    "    predicted = np.squeeze(np.array(outputs))\n",
    "\n",
    "    #if (i < 2):\n",
    "        #make_batch_plots(X,true,preds.exp().detach(),epochs, i,'test', outdir=model_dir)\n",
    "    make_batch_plots(X,true,predicted,epochs, i+1,'test',classes, outdir=model_dir)\n",
    "        \n",
    "    \n",
    "    counter += len(X)\n",
    "\n",
    "print('\\nPredictions finished in '+str(time.time()-start_time)+'s')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot predictions on test set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.expand_dims(testX, axis = 1)\n",
    "Y = testY \n",
    "add_norm_CO(X)\n",
    "#X = X * weight \n",
    "\n",
    "\n",
    "outputs = model.predict_on_batch(X)\n",
    "probs = np.squeeze(np.array(outputs))\n",
    "preds = np.squeeze(np.round(outputs))\n",
    "#print (preds)\n",
    "#print (probs)\n",
    "#print (testY)\n",
    "#print(testX.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rand_by_mask(mask): return np.random.choice(np.where(mask)[0], min(len(preds), 10), replace=False)\n",
    "def rand_by_correct(is_correct): return rand_by_mask((preds == testY)==is_correct)\n",
    "def plots(ims, figsize=(12,6), rows=1, titles=None):\n",
    "    f = plt.figure(figsize=figsize)\n",
    "    txt_colors = ['red','green']\n",
    "    for i in range(len(ims)):\n",
    "        sp = f.add_subplot(rows, max(len(ims)//rows,10), i+1)\n",
    "        sp.axis('Off')\n",
    "        sp.imshow(testX[ims[i]], cmap = cm.gray)\n",
    "        if titles is not None: sp.set_title(f'true={titles[i][1]: 0.2f} \\n pred={ titles[i][0]: 0.3f}', fontsize=10, color =txt_colors[titles[i][1]] )\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_test_with_title(idxs, title):\n",
    "    imgs = [x for x in idxs]\n",
    "    title_probs = [(probs[x], testY[x]) for x in idxs]\n",
    "    print(title)\n",
    "    return plots(imgs, rows=1, titles=title_probs, figsize=(16,8)) if len(imgs)>0 else print('Not Found.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_by_mask(mask, mult):\n",
    "    idxs = np.where(mask)[0]\n",
    "    return idxs[np.argsort(mult * probs[idxs])[:10]]\n",
    "\n",
    "def most_by_correct(y, is_correct): \n",
    "    mult = -1 if (y==1)==is_correct else 1\n",
    "    return most_by_mask(((preds == testY)==is_correct) & (testY == y), mult)\n",
    "    #return most_by_mask((preds == testY), mult)\n",
    "\n",
    "def worst_predicted(y):\n",
    "    mult = -1 if (y==0) else 1\n",
    "    idxs = np.where(testY == y)[0]\n",
    "    return idxs[np.argsort(mult*probs[idxs])[:10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correctly classified [random samples]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAABzCAYAAAAxOpeKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2de7Ad1XXmv4VsngL0AElIICQLISGEwBYBG0+ASrDH9lAYz8CA7cQmIQ6UYUw85UzsqWSO23ZSEMZjnHHIxNgOUAET8IQJA06AAQPiFSMLCRAPIfQACQk90QPxNHv+6L7onrW+e+4+j9u6gu9X5TK9tE+f3WuvvXb3Pb2+bSklCCGEEEIIIYQQ/dljV3dACCGEEEIIIcTwQw+LQgghhBBCCCECelgUQgghhBBCCBHQw6IQQgghhBBCiIAeFoUQQgghhBBCBPSwKIQQQgghhBAi8L6uz2A2CsDnkNKV3Xen4z58AsD3AYwA8COkdClpsxeAawHMBbARwDlIaUX1b98AcD6AXwP4ClK6vZZ+t4t8XRtWlL5OjV3nayvsJwBOB7AuNdLsAdoYyvH4FIAdAM5LjbSg+rcvAvjTqul3UiNdM/S9bh/5uj7k63qQn2tE62JtKK7rQ76uD/l6cHrxy+IoAF+m/2LW/cPoYJiNAPDXAD4JYBaAz8JsFml5PoDNSOkIAN8DcFn1+VkAzgVwNIBPALiyOudwRL6ujwF9bUUNvi65GqWfWvFJANOr//0hgL8BACtsDIAGgBMBnACgYYWNHrKedod8XR/ydT3Iz/WhdbE+FNf1IV/Xh3w9CL1wwqUApsFsIYA7AdwG4NsANgOYCbOPA7gVqXpSNvsagJFI6Zswm4YyyR6M8in5S0jp6Ta//wQAS5HSsur8NwD4NIAnXbtPA/hm9d8/A/ADmFllvwEpvQ5gOcyWVud8qM1+1IF8XR+XAphmBfe1FaWv+/4CZEXp69RI37Qi+jo12vY1UiPdZ4VNGaTZpwFcmxopAXjYChtlhR0C4FQAd6ZG2lT1706Uiein7fajBuTr+pCv60F+rg+ti/WhuK4P+bo+5OtB6MXD4tcBzEZKxwEAzE4F8KHKthxmU1p89ocALkRKz8LsRABXAvgtmH0ewB+T9kuR0lnONgnAC/2OV6F8uvbsbJfSWzDbAmBsZX/YfX5Siz7vSuTr+vg6gNmpUfraip2+To20fJBJ/UMAF6ZGetaKnb62YmBfp0bwdS5sTCa1sA9H5Ov6kK/rQX6uD62L9aG4rg/5uj7k60EYqp9Xf4mUlrdsYTYSwEkAboJZn3UvAEBK1wG4boj69m5Dvq6PX6ZGa19bsdPXVjT7OjXk6zaQr+tDvq4H+bk+tC7Wh+K6PuTr+pCv+zFUD4uv9Pvvt9BcG7l39f97AHj5nb8G9qe9v+qtBnBYv+NDK5unr92qqo7hQJRF5rmfH67I1/WR7eu+v1D1Z4j+0jSQT1ejfDWhv/2eDr9jVyBf14d8XQ/yc31oXawPxXV9yNf1IV/3oxcPi9sA7N/i318CMA5mYwFsR6n28y9IaSvMlsPsbKR0U/Xu/hyktKjNv+o9AmA6zKaidNi5AD5H2t0C4Iso6wDOAnA3UkowuwXA9TD7HwAmoiwc/WXmd9eNfF0fWb62otnXqZG2WmHLrbCzUyPdZKV61ZzUSIuG6C9NtwC42Aq7AeWrT1tSI62xwm4H8Bf9ipw/DuAbPf7uXiFf14d8XQ/yc31oXawPxXV9yNf1IV8PQvdqqCltBPAAzJ6A2eXk398E8C2Uye9OAP0LPz8P4HyYLQKwGGXxZrvf/xaAiwHcDuApADcipcUAALNvweyMquWPAYytisf/M8p3lFG1vRFlMfq/ALgIKf267X7UgXxdG6lR+toKe8KK6OvUGNzXVnThawBW2E9R3ljMsMJWWWHnV/YLrbALq2Y/B7AMwFIAV6FS9KoKnb+N8kbmEQDf6it+Hm7I1/UhX9eD/FwjWhdrQ3FdH/J1fcjXGf1LKfXyfEIIIYQQQggh3gX0Yp9FIYQQQgghhBDvMvSwKIQQQgghhBAioIdFIYQQQgghhBCBodo6o3PMVgA4HiltaPNzewG4FsBclHLU5yClFaTdJwB8H8AIAD9CSpdW9osB/BGAaQAOfuf7yw1+/wlA334r/4iUvtVW34YrQ+3rsu0IAPMBrEZKp1e2qwGcAmBL1eo8pLQQZqMB/ATlGLwG4PeR0hNt9W2YYkXp69Roz9dW2MkArgAwB8C5qZF+NkC7uQCuBrAPyiLoS1IjJStsDIB/ADAFwAoA/zE10uZKtev7AD4FYAeA81IjLWj/yoYX8nN9yNf1IV/XyK67B7kOwPEA3kQppHEBUnoTZgcC+HsAk1Hes/13pPR3HVzZsKOLuA6+To3oayvsEgBfAmAArkqNdEVlPxbA/wIwEmVcf75SttwTwN+iHIe3Uc6Dezq5tuGE8kd9vFt9Xc8vi+WeQkPN+QA2I6UjAHwPwGWkHyMA/DWATwKYBeCzMJtV/esDAE4DsJKcex5SOq763/B+UBwuvt7JJSgV4jx/3M+nCyvbfwWwECnNAfAFlAE+bLGiFl8/D+A8ANcP0u5vUC6K06v/faKyfx3AXamRpgO4C30KfOUc6Gv7h9XnhyXyc33I1/UhX9fIcFkXW9+DXAdgJoBjUN4I/kFlvwjAk0jpWJR7qX0XZnsO3WV0R01xfT6AzakxsK+tsNkoY/oEAMcCON0KO6L65x8B+HpqpGMA3Iyd++F9CQAq+8cAfNcKG5Zv4Cl/1Id83YtfFs22o5Rw/TiAtQDORUrrYXYPgIUA/g2An8LsWpR/yZlcffKPkNID1d5HPwUwCaVsrHXYk08D+Gb13z8D8AOYGZrlXk9AuanusqrvN1SfexIpPVrZOvz6Gti9fA2YHQrg3wH4c5RS4YMxC0D5V9aUnobZFJiNR0ovddjPjrEi+jo10normn1tRfR1aqQHqv14uvZ1319LrbC3W/T1EAAHpEZ6uDq+FsCZAP4Z5VidWjW9BuVGrX9S2a9NjZQAPGyFjbLCDkmNtKaTfnaK/Fwf8nV9yNc1snuti63uQX7e75p+iXJjbQBIAPav9oEcCWATyk3Ca2e4xDWIr60wq+Kxj6MA/GtqpB1V3+8F8O8B/CWAIwHcV7W7E+W2J3+G8h7kbgBIjbTOCnsZ5a+Mte57OVz8/F7IH/J1Hr34i8l+AOYjpaMB3Aug0e/f9kRKxyOl76L8leh7SOk3APwHlH/ZQdX+/urzN2PnQABm82C2kPzvNNKPSQBeANC379EWAGMHbFOyqrINxkdgtghm/wyzozPaDxW7k6+B8if1/4LydQ7Pn8PsMZh9r3p9BwAWoUzmgNkJAA7HzgWzbvYDMD81uK9TIx2fGjt9nRrc19Xnm3xthc2zwhaS/zFf5zAJZSz30T+ux/dLCmsBjO/3mU7mQq+Rn+tDvq4P+bo+dqd1cXC/mb0fwO+i3HMRAH6A8sHnRQCPA7gEKQ14QznEDJe4fsePqTGgr58A8JtW2FgrbF+Ur+EdVv1b//3wzu5nXwTgDCvsfVbYVJSvuR6G+hkufs7hXZE/5OvW9OKn1bdRvicLlO/V/2O/f/uHfv99GoBZ/X65OwBmIwGcjL4HhJRug9nmdz6R0m/2oH/dsgDA4UhpO8w+BeD/oPw5d1ew+/ja7HQA65DSr6q6z/58A2Uw7wnghyj/+vEtlL8qfh9mC1Euio8C2FWbE7flayt2+tqKZl+nRrrNip2+To1dE9fVe+3DbWNV+bk+5Ov6kK/rY/dZF/O4EsB9SGledfxvUf7C8Vso6/nvhNk8pLR1F/Rtt4nr1EhPWWGXAbgDwCsofdh3P/H7AP7KCvszALcAeKOy/wTlg/l8lCVJD2LX3IPsNn7O5d2SP96rvh6K93D7d/CVfv+9B4API6XXmlq3eu3TbB6A/cm/fA0p/T9nW43yL0CrqvqEA1EWPrM2fRxa2Qamf0JO6ecwuxJmB7Vd/D40DGdffxTAGdUD9t4oF+a/R0q/g/TOXz9eh9nfAfhaeTVpK4Dfq/pjKEWFlg3c6Vpp6evUaPZ1v4QSsGJgX6dG8HUOq9H8C2z/uH6p75WD6hWGdf0+095cqAf5uT7k6/qQr+tjOK+Lrf1m1gBwMIAL+rX5PQCXVq+zLoXZcpS1jbW+GjkAuyqu3/G1FQP6GqmRfgzgx9X5/wLVrzKpkZ5G+dohrLAjUZbL9P1K+dV+fXoQwJIBO10fyh/1IV8TevGwuAeAswDcAOBzAO4foN0dAP4TgMsBAGZ9wib3VZ/7Dsw+CWD0O59o7696twD4Isp3hs8CcHeooQMeATAdZlNROuvc6rsHxmwCgJeQUqpejdwDJCnVxO7j65S+gfIXRFS/LH4NKf1OdXwIUlpTPRCeifJ1EcBsFIAdSOkNlMX99+2iv54CHfraCjsuNZp9bUWzr4fgL6hrrLCtVtiHAfwrSnGg/1n9c99YXVr9/z/1s19shd0A4EQAW3ZJvZH8XCfydX3I1/Wx+6yLre5BzP4A5a+Iv+1eM30ewG8DmAez8QBmYNf9EXW4xHXwtatXRPW946raw8kof/35sLPvAeBPUdaioXpd1VIjvWKFfQzAW6mRnmyjX71iuPh5UN6r+eO95ute1Cy+AuAEmD2B8jWJgdRCvwLg+KpO7UkAF1b2AsDJMFuMcjI/32E/fgxgLMyWohRTKZWAzCbCrCwcL+sILkZZzPwUgBuR0uKq3VdgtgrlU/djMOt7H/ksAE/AbBGAv0JZPL+rfkrffXzdmutg9jjKV00PAvCdyn4USl8/g1LB6ZIO+9cLXgFwghV5vrbCHrMi+tqK7nxthf2GFbYKZV3F31bn6/u3hf2afhnlO/RLATyHsuAZKBPHx6ywZ1G+RnFpZf85yhuOpSiLu7/cSf96gPxcH/J1fcjX9bH7rIut7kHKB5bxAB6q6iL/W2X/NoCTqjXzLgB/sgvfbBoWcY3K11Y0+9oKm2hF0z3I/66+//8CuCg10suV/bNW2BIAT6OsBe3bimQcgAVW2FMoy2N+t8P+dcuw8PN7KX/I14P0r+vnHrPtSGlkdycRWcjXtWGFbU8N+XqokZ/rQ76uD/m6RrQu1obiuh7k5/qQr/MYlvvHCCGEEEIIIYTYtXT/y6IQQgghhBBCiHcd+mVRCCGEEEIIIURAD4tCCCGEEEIIIQItt8648cYbwzuqI0aMaDrea6+9wufe//73Z325b8c+9773xS7usUd8xvWffeutt0KbX/+6s71V33zzzWB7++23g+3UU09tsWFTa66++urg6z333LPpeJ999gmf23fffYONtfM+23vvvUOb/fdn28FEXn/99ZbnBnhc+M8BwKZNm5qOX3nlldCGjdvpp5/esa8vu+yyQX3N/MPik127JzeGmc3HGWvDzs985j/L4nr79u3BduGFF3bs6yuuuCL42vfX5xSA+5/FlG/HxojZ2LzxttxzsTHx18TaMP/PnDmzI19ffvnlg8Y0819OLmU2lv9YHBrZw87bWCkEs73xxhvB5vvB+sXWgm5i+pprrgmd8zmX5WB2TTl+ZPODjRHztT9X7ueYz3y87tixY9A2APCZz3ymY18/8sgjwWmjR49uOt5vv/3C53LyMhDXpJdffjm0efXVV4ONjZufXyzHHHDAAcHGxtf3a8OGKH66bdu2YDvllFM69vX1118ffO3j5eCDDw6fGzNmTLCxe4mc9ee1114LNraWef+wewt2Lvad/vzsXCzWv/CFL3Ts65tvvjn42scUm4MsB7L+5tw3sFzA4tPH8dixY0ObUaNGBRubl369Zus8sxlLUhmMGzcu+Pm8885rOj777LPD5+bMmRNsbP30cZETXwCPTX/vy9owG4vNrVubd5xjuYLZLrjgggH9rF8WhRBCCCGEEEIE9LAohBBCCCGEECKgh0UhhBBCCCGEEAE9LAohhBBCCCGECLQUuMkRnGGF/F5YYSB8zSr7PlZInkM34hT+O9nnWCFrN+QIqrCCeVZEzPDXyYqnmbgMG8scASN2fjaWI0eObNlPgIsOdAOLWR/XTKyDwcQJcsgRRWG2XPEoVg+es6dq7tzNJUdogl13rkCFn4e5IlZs/ubM6U4FYdi5mTBBp7Bx837N3VOXzV1/LtYmV4Mg51y5ed/7mQkA9HovYZZzc+ZprqiVt+UKv7HY99eeOz/YnPRj0utcwfBiNkD0P+sHi58coTrm1xzBOCCKgxx44IGhDbOx+Fy3bl3T8ZYtW0KblStXBtspp5wSbLnMnTs32LxvWSyy/rM57ecmE+bIFarxtlxhERYX/jvZ9/UyVwP8Xs7DxGxY3OWs9bn5lN0D+jnB/MP6yuZSzn0V62un91kTJ04MNi/Qw+69WezkxBjzQ65QjT8/8wOz5eT0XJG/VuiXRSGEEEIIIYQQAT0sCiGEEEIIIYQI6GFRCCGEEEIIIUSg5QvEbPNNvzFlbg0Ve4c9p5YkZ3Nxdv7cfnW6EXqn71APBKuL8O9S59bRsRoO7x/2bjqri2D98u865/hwoH7l1L30uj6Uba7q44X5ml1nTv9za7ly5gPzRW59n3+fnr1fz+pIuoHNw5z6rpyNqoG89/W3b98ebCwGfFyzjaVZzVFOzQMbNzbfOiWnji63do/N3U7rjdh3en+xsei0Fp6ND4vzbmD1Jj6GWUx3ek3MP8yWkwdYLspdK3NioNe5ms1dP56d5j8AePXVV5uO2TWx87M579cMNifZfGD98rlh9erVoc2SJUuCrRtYfZePF5YbvA8BPkf8JuCbN28etA3AY8DnV7Ze5N4v+bzFYpjdv3YDywWe3Jo/lkN8HLMxYteUc+/I1tzcGnZ//pxa+24YM2ZMsPn+sznP4pddt49Xpq/BYprNeT+32Pfl1uF6HzI/t5ur9cuiEEIIIYQQQoiAHhaFEEIIIYQQQgT0sCiEEEIIIYQQIqCHRSGEEEIIIYQQgZYVwKw41BdK5hajsiJMX+TLzsUKOlnBsy/gzSkgHeg7fRExK9TNLZ7OhfXNfwcTNWCfyxHOyBWlYYXRvrifiUrknssXRjO/MqGAbmDF5d6WuxE2w4sYMFEDJgaSE7O5YjAsVnz/2aasvd5om/XNXwOLRVZgzgrRfS5gBebM1+w7fb479NBDQ5vJkycHW45gErueTZs2BVunsOvx483ikH2OxeHGjRubjtevXx/avPTSS8HGfO/nMxNSmzBhQrCx9cjnolwxqW5g4ib+mliOyRWn8O2YD/14AFywwMchm/N+M3kgb96yuM8RnGoHFlP+Gtg1sfWH5Q8vPsHuLZh4x8iRI4PNzyXWJve+xLdjeW358uXB1g0sZv18Yj5kNhafL774YtPxqlWrQpu1a9cGGxsTP0dYDOeKkflrZHkyd1P7XHLERVisMNvWrVuDzYsHsXzNPsfm/bhx45qODznkkNCG+Yf52rfLFQ7qVPSG5U4/l5jYI5tvDC885WMc4HOBfae/Rnbfy0T3WDufi1hebten+mVRCCGEEEIIIURAD4tCCCGEEEIIIQJ6WBRCCCGEEEIIEdDDohBCCCGEEEKIQEvFDlY46YvcWZEkE01gxfD+/Ey8gxWassJcXwTNBCUYrF++MJe16TWsGJj50cMKV1nRsC+yZf7JFXB54YUXBv0cE60YPXp0sPnrZsXyOX5oB9ZfH8e538li1hevswJnVsjPBIC8SAIrLmdFz6zg3xeXs+/rNTlzhxXHM1EDJnbhRR5WrFgR2jDBJxYDEydObDqeOXNmaMPmG4sBH+tM0MCLEHQDm1s5ogwsBpi/Vq5c2XS8bNmy0IblajaOXnRg0qRJoc306dOD7QMf+ECweSEclvt6LU7BYicnX7C+MQEGL37w/PPPhzZMSCEnf0ybNi20YTklR1QsV4CjG7yABBBzG8t/DCYq4XPKunXrQhvmVyae4teu3DWEtfNzic2tXuYPgMe1z3dsfWZ98/cIAPD44483HT/22GOhDRPtYXPE+//ggw8ObVheGT9+fLD53JkrtNgNnQokshzC4vPZZ59tOn7qqadCG3ZfwgS3vB9nzJiR9Tl2D+j7z9bFXopHsn758WVj64WvAJ4bnnzyyabjxYsXhzZMyImNmc9jLH6nTJkSbP6+BQBGjRrVdMyeL5itFfplUQghhBBCCCFEQA+LQgghhBBCCCECelgUQgghhBBCCBHQw6IQQgghhBBCiEDLSlJW6OvFHHLFA1hBpxc/YG1YgTsTsfDF3qzvvtgf4EW4vniaFaAzUYtuyCmGZ4W/rPh7/fr1web9yIrjmegNa+fHiYkhMMGNww47LNi8aAUrQO81bOx8IXRKKetcrEDbiybcf//9oc3SpUuDjQkF+Dj+4Ac/GNqcdNJJg/YTiPHDYqfXYiA552PCE5s2bQq2Z555JtgeffTRpmOWGxgHHXRQsPnie1b4zsabXeOsWbMGbcOK6DuF5Ts/3kwcgV0PE6fwgjZMuIgV2q9ZsybY/PxgQgss1zEhFj9PWS4ys2Drhn333TfY9tlnn6Zj1n/WD5ZzfQx7YRCA+4wJMPgYZmILc+fODTYmMJQjesPiqRvY+byIDssfLNZZfvXxyfzD5ikT1fH9YJ9johVMnMWvsT6+BvpcN7BY9POL+TpH+AMA7r333qZjn7sBLojF5psX4dqwYUNow+bIEUccEWweNra59wPdkJOnmPgYE6958MEHm45ZDmFzi127z9dMYJIJyYwZMybY/D04y5Ps/oytbzmwc3kbE21iomLMh/fcc0/T8a9+9avQht3LsJj2ojRM7MkLzQHA7Nmzg83nb/ac065P9cuiEEIIIYQQQoiAHhaFEEIIIYQQQgT0sCiEEEIIIYQQItCyZpHV+Pn3XNm73Oy9c1ZH59+FZu/osjolVlfgaxTYpuSsXoC9p58D8003sBo/X8fI+pq7qbavWWQ+ZPWJOe+ns1oiBqsj8bB3zMeNG5d1/lzYd/h351kNKXvHm/nf13yx+cA2tmU1Fg888EDTMaszZHGds9E2qxdg7+93A/Ojr5Vg18RqUNj7+r6+i43tMcccE2yHH354sPl6HFZr8NxzzwWbrzUAov9ZvuhlfSibW74GhuUPVse1du3aYPMb+PqaTIDXLLJ+3XTTTU3Hvr4G4LWnM2fODDbv515vns1gPut0HWEx5muQ2Eb3vtYb4HUwv/jFL5qOmV9ZrRQ7lx9fds29zh/sfD4PsxzD4i6n3s5vZs7aAHzuep+xMWK1oKyW39eOsRrhXtf3s7j2az1by1588cVgY3V03sbyPlsXWX25Xx9YzsqtCffj1suauYFg8eO/g817ti4yX/v6OlafyOKT3Tf7e0BW075o0aJgY/e03ub1QXoNq5v0tb5svNk1zp8/P9gefvjhpmMWc8zPbK30awF7ZmL50K/NQLwnYeOqmkUhhBBCCCGEEF2jh0UhhBBCCCGEEAE9LAohhBBCCCGECOhhUQghhBBCCCFEoKXADSvC9TbWhol+MPGUJUuWNB2zDS1Z4SwrOPdFqpMnTw5tWCEoEwLxBfNsQ1xWcN4NrNDX94MV7bOCcyZw44v7mXANExNiBbsnnnhi0/FHPvKR0IaJp7CN6H0hMdtomBW4dwPzo7exomcmnsFi0V8nu6Zzzjkn2FgMXHTRRU3HTAyEbYA8ZcqUYMuhU6GOdvA5gwkdMLEFL4jFmDZtWrCdccYZwXbyyScH2+LFi5uOb7vtttCGjTcTePDF6kwwhNk6hQmNMYEED4tpNj98gTwT5TjttNOCjYlweN8sWLBg0DYA97OPcyY2xPJ+N7C1LGfDerZWss2fff5mYkJnnXVWsLGNsb/61a82Hd9yyy2hDRPIYKIMXpyCrYu93ryczRE/nuy6megNs/nck5t32Prp16mjjz46tGF99eJzQJw3TCyE5bpuYHPOz3smEsSEONg1eV8fddRRoc2ZZ54ZbCzXzJs3r+n4rrvuCm3WrFkTbGzTeT+f2Xrd6xySE58sD7Pcw+LT33/NnTs3tGH3ICxf33777U3HDz30UGjDxCnZuHkBI9aGifF0er/NYuzII49sOmZjwdZ5JnrjOf7444ON5WomdOVj+tZbbw1t2FxbtmxZsB166KFNxxMmTAht2FxohX5ZFEIIIYQQQggR0MOiEEIIIYQQQoiAHhaFEEIIIYQQQgT0sCiEEEIIIYQQItBS4IaJXfgCbSaQwApUWWG0L6hmBcmsUHrDhg3B5gvhx4wZE9qw4nsmxDJixIimYyZ2wmy9xgsF5QizAHzcvFDNxo0bQxsmjMPO72Ng0qRJg7YB+Fh6EY4dO3aENr2GjbkfTx8DAI91Vhztr33kyJGhjS9ABrhggRe3YGI2zNesIN8LUuSMbbcwUQ/mx5zPMZsvfB83blxow0SsmP+9eAATHGK5h803nwOZUAfra6cwAQAf08zvbC4wP/t2TNyExY4XyACAsWPHNh0zASs2Z5jwhD8/i/tei1Owa/ewvMBga6W3sXMxIR8mYjBx4sSmYxYnTAiJiU7lrEfsc93A1lkfZyzu2BgxMTg/x5lfc9dKP0eYSNARRxwRbCyn+/ulqVOnhjZs3e0GNnY+Z+SOOWvnfe1jE+B5mYm1eUEVNm4sV7Pc5ucXy4m9Xhdz7h/NrOPz+/tfFj8nnHBCsB1zzDHB5mPdC8EBXGSHifH4+zsWJ70UyfrQhz4UbH4dYaIxTNiRrfN+XWf3Gix+jzvuuGDz4lrz588PbZjIW05f2f0ss7VCvywKIYQQQgghhAjoYVEIIYQQQgghREAPi0IIIYQQQgghAnpYFEIIIYQQQggRaClww4pPO4UV6/pCYiYCwYQOfKE9a8eEFdj5WeGyLz7uptA4F1Z47fvBhBpY/9m4+cJiVsTN/ONFiABg6dKlTcd33313aMMKzlkhri82ZiIWvRaoYOIEXuSBiYEwGxNN8Od67LHHQpurrrpq0M8BwMKFC5uOx48fH9qwWGfx5IUzcuJkKPBxzYr92ZgzcQ5fpM2u24shANzXXoCJjS3rFxPC8XOJtWHX0ym5gioeNudZPPni+2XLloU2LKc89NBDweYL95lvmPiFF8YBYv/Z9fQa5msWdx6WP1gMeCFvcXQAAAsZSURBVMEQlj+uu+66rPM/8cQTTccsfzBfs375a2TrYq+FQJiv/RxkfWW5jcWnt+23336hDVuTGF5U57nnngttWH5lwhl+nOoQXWH98EI7uesD668fSyZ6yMRTVq1aFWw+/7B+5dxzsn61K/zRK3w/WFzvv//+wZaTr9l8YMJKTOzFr5/sXKyvLD79NTJxJBaH7LpzYKJf/n6D5Zgc4Usg5kR2z8Bi2o8PADz99NNNx+x+md0rMd/45yH2uXbvGfTLohBCCCGEEEKIgB4WhRBCCCGEEEIE9LAohBBCCCGEECLQsmZxzZo1wZZTI8Le92V1KX4DVvYeLzsXe9fd1xqwzV3ZJrk5NWBsg99ev9eeUwfD6qxYP0aPHh1svoaQvRc+c+bMYGPf6fu1YsWK0Ia9K878f+SRRzYds/qQTuuxBiKnVo99Z44vgBh7bCPVW2+9NdhYrPt3zdmmuWyjZzYv/XWzDcFzaq/aIWfsWL0Di2EWU75GgOUsVjPA2m3durXpmNUM5OQxZmMbz7Mas05htcV+LNn3sfpvVrflY3jBggWhzbPPPhtsrJZo3bp1TcfMpywvs5j2+a+XPh2InI29GayGivl63LhxTce+RhwA7rjjjmBj1+797/MtAEyfPj3YWH1ojq9ZTukGlnO9Lbf2l42Rv87c+s1JkyYN+n0sTti6yzby9jZWc8TyUzewsfN5hfU/t47Kw9bF3Pp1vwk80yFg5/Kb1QPx3ojdU/VSvwPgvvbfy2r+2DrCcuXKlSubjln97IMPPhhszGc+r2/evDm0YfkiR3+CxRObp7l1wx5Wz+nJ9bPPywCwdu3apmN2X+F1JwBgyZIlwebXRTa/mRbI1KlTg83nDzYf29UC0S+LQgghhBBCCCECelgUQgghhBBCCBHQw6IQQgghhBBCiIAeFoUQQgghhBBCBFoK3LDCzMMOO6zpmBWEs2JKVqDqBT38RpLs+wAucOPPxQqZWSEwK2T1ogOs4DalFGzdwK7JF1WzgvlcMaGjjjqq6ZiJhbAibvadvjCWFcoyGysQ9oW4bNx6uXk5wAuH/XewoufcTai9bz/60Y+GNi+88EKwsZjyBdpMWIHFNcMXkzPxCCZ40g05QlBMuITllTlz5gx6LuZXtvkwE9Dx/WDziIlAHXfcccHmxZxYnum1GMhgsLFgvmeCPX7zZCZqlStY4X3PivaZEAvLWX5OshzWa4Estk75fjDhBgZbK48++uimYybSwHzNNrj2PmM+ZDmFibr5eO21Xxksv3obW2vY59hamSMCxeZuzrrL4prlnRzxCRZPXpCrW9iG4l4gKWftBIDDDz882DZu3Nh0vGrVqtDmmWeeCTbmaz/PWZ5heYzlFb9+suvpda5m91V+zrHrZmJIbF1cv3590zETuLn33nuDjX2nvydgucHnLIALZ/l7a5avc3Nnp/j7HpbHWJ6cPXt2sPk5uGzZstDGxz2QJwDF+jVt2rRgY6KHU6ZMaTpmuahd9MuiEEIIIYQQQoiAHhaFEEIIIYQQQgT0sCiEEEIIIYQQIqCHRSGEEEIIIYQQgZYCN0zEwBdmMgEaVgDLitC9uAIr+mTiHUyYw8MKnpmYQ454Cvs+VgjfDazw3Rcbs8LfHAEAIF47K3hlAi6snRcrYuPG+sDa+X6xz/UaNub+e3NFdVic+fhnoj0zZswINlbs7WPAF4gP1Ad2Lm9jxew5gjTtkDNP2Hey+eBFmoAYU7kF5jkCTOz7WDH5rFmzgs3PEVaszsatU1iO8vOZzW8mqsR84/M8mx9MpIud3wvEMCEhJurC+uVjh4m89DpXs/XN+5aNLfMFy69+zWNiT0yMhIlm+H6w/MHGMke8hn1fztrcDjlrNusrs7GY8tfAYix33fU5i31frnjKpk2bmo63bNkS2jBbNzDRpA0bNjQd5wizAPy+0M8RluPXrl0bbGz++rhg94lTp04NNiZ64/MRGw+WV7qBxYG/JrY+M6HIuXPnBpv/7Pz580MbJnrD8ooX1WHr3bHHHhtsY8eODTY/lmxusXuXTmFrkh9fdv/B/MwEbnKENdm8YjHt8wUTiWJ9YEJCTEzSw+ZyK/TLohBCCCGEEEKIgB4WhRBCCCGEEEIE9LAohBBCCCGEECKgh0UhhBBCCCGEEIGWAjerV68ONl90uW3bttCGFVeywnFvY0IHrEiWFeF68YBXX301tGEFnTnFtL0WSGDkCPKwgnBmY0IyXtiAXRMTP2Dj5tuxYmBW3M/wY8nGo91C3MFgxepMfMLDitJzxCFyhWSYAIm3MWEF1nf2nb4dK/7OEbZohx07dgzahsU+s7H84P3vi/EBHj9s3Pz5mRjCpEmTgo0VtXs/9jqGPWwsve+ZcAODxY7PA5MnT846FxtHf342Fkw0g8W+z2PMzzkx2A5sbfF9y53fDN+OjQcTfsk5PzsXy7k5OYWNbR35Iye/Mph//LixHMPWwBxRHbbGsnnKRD58Oy80M5CtG+6///5ge/7555uOmS+YEMeECROCzecMdp+4devWYGN+9DmD3fOwscwR0GNxkhtjuTA/+r7l5GGA5wJ/j8bWMia8wmLRn5/d77E1kPXLz5Ghvt9j8y1HxI+1YTHt/cyEnV5++eVgY9foY5j5mQmisfv2HNrN1fplUQghhBBCCCFEQA+LQgghhBBCCCECelgUQgghhBBCCBFo+SI2e1fc172wd2/Z51gNhH/vPPdczJZTe5i72ae/xtw+dAOrP/B1O7kb1rN33X0tF/MFq2nK2cCevRfOxpK9b+9h9UCsboW9I98NOe9vszY59Q3d1C/5eZO76XXOe//sXL2Oa3a+nJoBRk5dL6tZZPOB1cTlbBbPbKxfOf7v1A8M9n2+loHlYDa3cjbezsnnAJ8zOfU/7HOdxmZO3mkH1g+fE1muZnM+Z03KjRNWi+3HMqduD+D+9+fqdGy7xfeDrRm5a70/Fxvb3Noe7//c2is2br52LKdNt9x+++3Btnbt2qZjtu5u3rw52GbMmBFsU6ZMaTpmehSs5qvTmMq9n8yJldz1uhty4oX1g61lBx10UNMx8yGriWP3BL2c9/5c7Jp7uS6yOdJpTSRb530NIavTZPfV7Bq9X9m6xT7H+jUU6JdFIYQQQgghhBABPSwKIYQQQgghhAjoYVEIIYQQQgghREAPi0IIIYQQQgghAi2rUqdOnRpsXkCCbQrPCjNzRCBY4WluMWrOpuq558oRuMkVGsnFb04PxA1qWRHxAQccEGysCDZHmIht7snED3xBOPMFKxrPKdLfsmVLaMNEOKZNmxZsubACbR8/uZvFsjHx/mffx3zGCqFzCutz++rjmPWBiUV0Q67QRKf4gnKWe5jwCmvnbSxnsVzAiuj9WLLCd0anm5rniAIwP3S6ATUbV/Y5ZuulWIQ/F1sHeh3TDB8XbBxzY8D7Nnfdyomd3DWW2bxvc4XsuoGtGd4/bH1gvsgR8sn1BYvrnHFibVge3rZtW9Mxi+FOc8VArF+/Ptj8/GJzd926dcHG8pH3Gbvf8MIsAJ83Ps5YnDBbjs/YOtzruGb3OT4O2HXn3I8BcU3qJob9udj3sRjOEcvJ7Ven+HmUC1v7WUz7HMja5AqIdUpO3umFcKF+WRRCCCGEEEIIEdDDohBCCCGEEEKIgB4WhRBCCCGEEEIE9LAohBBCCCGEECJgvS7cFUIIIYQQQgix+6NfFoUQQgghhBBCBPSwKIQQQgghhBAioIdFIYQQQgghhBABPSwKIYQQQgghhAjoYVEIIYQQQgghREAPi0IIIYQQQgghAv8f8HkWlVlfmowAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A few correct labels at random\n",
    "plot_test_with_title(rand_by_correct(True), \"Correctly classified [random samples]\")\n",
    "save_name = f'Correctly classified_random.png'\n",
    "plt.savefig(save_dir+save_name, bbox_inches='tight', dpi=200)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best predicted bad CO-tips\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAABzCAYAAAAxOpeKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfaxl1Xnf8d+CMYNthhfPMAaGgZnh/YIoYFqGWim2iVo3kYhdU8mQSLaUyk3stn+krmwrlWLlH1eNkjRS0ubFrRqlUQ1Ja5k2iR2HYBtMHcAyYzODgQEGw/gNz3iGmQGDwat/3A2c86zfvfe55+y77/Xx9yNdcc+affZZ+9lrr7039zzPLrVWAQAAAAAw6rjV7gAAAAAAYO3hZhEAAAAA0OBmEQAAAADQ4GYRAAAAANDgZhEAAAAA0OBmEQAAAADQmP5msZRTVcr7e+jLNH14u0p5SKXsVSkfXmCZ9Srllm6Zv1Mp20b+7SNd+0Mq5Z8M0+kJEOvhEOvhEOvhEOthEOfhEOvhEOvhEOvhEOul1Vqn+5G2VemBBf5t3dTrX/rzj6/So1XaUaUTqrSrSnNmufdX6fe7399dpVu63+e696yv0vZuXceveL+JNbEm1sSaWM9mrIkzsSbWxJpYE+sZiXUfX0P9D5LOUyn3q5TfUClvUSl3qpTbJO1RKdtUygOvLF3KB1XKR7vfz1Mpn1YpX+7ec/EEn/8PJO1VrY+p1hckfULSz5nlfk7SH3e//7mk61VK6do/oVqfV62PS9rbrXMtItbDIdbDIdbDIdbDIM7DIdbDIdbDIdbDIdZLWNfDOj4s6TLVeoUkqZS3SLqqa3tco38mbf2hpF9SrY+olGsk/WdJb1MpPy/p35nl96rWG0PbFklPjrx+StI15r2vLlfriyrlsKSNXfuXwvu3LNLn1USsh0Osh0Osh0Osh0Gch0Osh0Osh0Osh0Osl9DHzaJzT3d3u7BSTpL0DyX9mUp5uXW9JKnWP5X0pyvUt1lDrIdDrIdDrIdDrIdBnIdDrIdDrIdDrIdDrEes1M3isZHfX9R4IZ0Tu/8eJ+nQK3fyo5Z3R75f0taR12d3bdHLyz2lUtZJOkXSgWW8f60i1sMh1sMh1sMh1sMgzsMh1sMh1sMh1sMh1qN6SMzcWKUnRl6/pUr/d+T1a6r0vW659VX6UpU+2v3b3VX6593vpUp/b4LPX1elx7qkzpcTQy81y30gJIbe2v1+aUgMfWwNJ+ESa2JNrIk1sV7rsSbOxJpYE2tiTaxnJNbTF7ip9YCkL6qUB1TKb5h//6GkX5d0j6TPSvr6yL/+vKRfVCm7JO2WT+hc6vNflPSvJH1G0oOSblWtuyVJpfy6SrmhW/K/StqoUvZK+hXNf0dZ3bK3Stoj6dOSPqBaX1p2P4ZArIdDrIdDrIdDrIdBnIdDrIdDrIdDrIdDrJdUaq19rg8AAAAAMAP6eHQGAAAAAGDGcLMIAAAAAGhwswgAAAAAaKy9m8VS9qmUTRO8b71KuUWl7FUpf6eFHqJZyttVykPdch8ead/evW9vt54TlrXeH0fEejhrL9a/olL2qJSvqpTbVcq5k2zWmrT2Yv1LKuVrKuV+lXKXSpmbZLPWHOI8nLUW61f//V0qpaqUq5fdt7VqrcW6lPeqlKe7cX2/SvkXk2zWmkSsh7P2Yn2OSrlDpXyluw75mUk2a81Ze3E+V/PXeF9VKZ9TKWdPsllDlaVdt4xl91Vp0wSf8f5QUvYWs8zxVXq0SjtGytPOdf92a5Xe3f3++1X65fR619IPsSbWuVi/tUqv637/ZWK9orE+eeT9N1Tp06seT+K8+j8/zrGef72hSl+o82Xkr171eM5qrKX3Vul3Vz2GxHrt/fx4x/oPR36fq9K+VY/nbMb5z6r0nu73t1XpTyaJQR9BPFql367S7irdXqXTu/bPVek/Vem+Kv3bKp1epf9VpXu7nzd3y22s0l937/94lZ6YMNCfqdK1r+zY+WeilLDMtVX6zMjrj3Q/pVt+XbNcZr3DDVhiTaz7ifX4+6+s0hdXJc4/ebG+qUp/RZxnOM4/KbGe346f7bZp9W4WZz3Wa+kGhlgT6/5i/QdV+tBI+93EeUXivLtKW7vfS5WemSROfXwN9fWS7lOtl0r6vKRfG/m3E1Tr1ar1NyX9jqTfVq1/X9K7JH28W+bXJN3Vvf+Tks555d2l3DnydYDRn582/dgi6cnuz6UvSjosaeOCy8x7qmvbKOlQ977R9ux6h0KshzPrsR71i5L+auFQrLjZj3UpH1Apj0r6j5L+TSImK4E4D2e2Y13KVZK2qta/yAZkBc12rOe9q/sa2Z+rlK2JmKwUYj2cWY/1RyX9gkp5StJfSvrXiZishFmP8y5J/6z7/Z2SNqiUZV9Xr1vuG4wfSbql+/1/SPrfI/92y8jvPy1pTqW8/PpklXKSpH+klzek1r9QKd9/5R21/lQP/ZslxHo4PxmxLuUXJF0t6bpV7MXsx7rW35P0eyrlZkn/XtJ7VqEXxHk4sxvrUo6T9FuS3ruq/XjV7MZ63v+R9D9V6/Mq5V9K+mNJb1ulvhDr4cx6rG+S9N9V62+qlGsl/YlKuUy1/mjgfsx6nD8o6XdVynslfUHSfkkvLXclfdwsRnXk92Mjvx8naadq/cHY0q8GvlXKnZI2mH/5oGr9m9C2X9JWSU+plHWSTpF0YIFlXnZ213ZA0qkqZV13Z/5ye3a9q4VYD2fWYq3u/279qqTrVOvzC3d4cLMX61d9QtJ/WbjDgyLOw5mlWG+QdJmkz3X9PEPSbSrlBtV638IdH8wsxVqqdXQdH9f8X83XCmI9nNmK9fw3mt4+v2X1/6mUEyVtkvTdhTs+iNmKc63f1Ms3s/M3t+9SrYcW7rTXx9dQj5N0Y/f7zZLuWmC5v9bon5lLuaL77Qvd+6RS/qmk015ZptafUq1XmJ8YZEm6Ta/+X+QbJf2taq1hmXslXaD5qkEnSHq3pNu65e4Y2Y73SPrUMtY7FGI9nNmOdSlXSvoDSTeo1tWenGc91heMvP9nJT2ywPatNOI8nNmNda2HVesm1bpNtW6T9CXNzyOrdaM4u7Ge79OZI++/QdKDC2zfEIj1cGY71tI3JF3f9e8SSSdKenqBbVxJsx3nUjZ13waRpI9I+m8LbN/iekoO/a0qPVClvw3JoVePLLepSrdU6atV2lNfrfozmhz6R1Mkh55Y56v+7K3SPVXa0bWfVaW/HFnuZ6r0cJ2vKPSrI+07uvft7dazftH1rl4iLrEm1n3E+m+q9J0q3d/93EasVyzWv9P17f4q3VGlS4nzDMf5JyHW45+xFgrczG6spY91fdvVjeuLiTWxnoFYz1Xpi12s76/SPybOKxLnG6v0SPeej1c3hyd+Sq11opvMV5RyVLWeNN1KkEKsh0Osh0Osh0Gch0Osh0Osh0Osh0Osh0GcU/r4GioAAAAAYMZM/5dFAAAAAMDM4S+LAAAAAIAGN4sAAAAAgMaiz1m84447lvyO6nPPPde0/fCHP0wtN7TXvOY1TdtrX/vaJZfLLCNJ11xzzSIPXFncxz72sSbWxx239L28W6Ys9tyXHsSvLv/oR7lnqGa2x3Hr/9CHPjTxRn72s59tYn3CCSeMvT7xxBOb961fv75pO/7445dsW7euPczc+9xy0Ysvvti0ufi4YzC2vfDCC6n1X3vttRPH+vOf/3wT6xhHF9ds/OOYcrHItr300kuLvpZ8XN1ycf0u1m5db33rWyeK9Z133tnE+fWvf/3Ya5dy4Oax559vH7kZ45ydz3/wgx80bXGcu33hjg/X/8wx49Y1zZg+fPjwkudFt03u2Mocp1mZ+cnFMNvXOC7cGHBj501vetPEsXbXIPF87MaAmz/cWI/cWHFxdXHMzEWOO0bifOGWcfto586dE8f6ne9855Kxvu6665r3XXHFFU2bi1mcJ5999tlmmaNHjzZt7niIsXWxzswN2XW5a5d3vOMdE8f69ttvX/K86K47XVzddsaxnt2mTMyy15fuvOjGbOTmkMsuu2yiWN96661LztVuXtiwoX00optT4jnWxc/tswx3zeDm3CNHjjRthw8fHnvtjiG3L26++eYF48xfFgEAAAAADW4WAQAAAAANbhYBAAAAAA1uFgEAAAAAjUWzWV0CZExKzha4cV73uteNdyaRqLuQ+JnTFBOIn+mWccnHfYsJyC6x2MWsz0IymeVcUYDsuqLVeu5nJmnbLZMpVOPGcLYwUWZdbqxnjiWXeD1pcY2FuL7Fz80WAHImHWdO5rjJjs/MurIFGDJior3UJum7fZtNvo9xduPQtbn9GItJuXOIW1em/5niPNOKxQMcN07csZApjuS4bXLrj3HMjt9MX926+p6/TznllKZt0mJk2fk7w8U/c150cXX9z8z7rkDMNDLxcf1wsY7XdlJbpCfux4XaXKz7nF9dIZGo7/NipqhbtphNZiy697l5xo2BeExPc4y7/buS3L7NzB+ZYjZSG1e3jLtXyIxft3+OHTs20bpcEZzl4i+LAAAAAIAGN4sAAAAAgAY3iwAAAACAxqJf6nbf0475JZPmJ0rSySefvOhrafK8nmeeeaZpc7kxrq3vXICMTA5b9vv8Llcik2/i3jfpd9gnzX90OTx95xy5XID4vfJJH+ostduUPUZc/ON7s8dD5uG32bzJabjv62e2wY2DjGx+SyZfI/vQ4sz2uHX1mQeTmaunye+Mx0M29yTzYGY3f2QfIhy5h5f3nUfn8iLjGMg+BNstN+lDwrM5kVE25zyu331eZh8th+tbnIenyYOK/c2c76Rc7pjbR25/uM+M8XfL9H1edLZu3Tr2evPmzc0y7trOnT/jNrmxkt2XMf6ZcZJdVyaHcVqZc4Ybd26b3LrieHHLZI/VzDVB9rzYZ42BDJfzHPMKY26/5K9b3P1JZi5y15uZa3kXPzc2XT5ibHN59svNY+QviwAAAACABjeLAAAAAIAGN4sAAAAAgAY3iwAAAACAxrIL3GSKMmSK2UjSG97whiXf52SSg10isCt648RtHKLgTaaYRuah7QuJy/VZ9CGbtO8+M5OkP2mxk4VkEo7d+HFJz5nEercul+jtkpfjWHTFNZy+C9VMKpPQ7vrqYuaSwuM+ycb66NGjTVucH9xcly1Skimu0Ocx6JL0Y1/d+HXcg4Xj+t0x5ObXAwcONG0HDx4ce50t4OL6H+PsCnD0Lfuw7EnXFbfdnRddfDLF2twy2Tkl7qdsAahpuPk1cuMn8z4p9yB0x8UsFlea5rwV++HW1fcDzt/85jc3bXNzc2OvL7jggmYZN/e4cRbPb+66yp0DM/PDSSed1CzjjhsXs7h+N4+5AiHTcOMnbtM05/B4HGbnLFf0JnPNnymIJeWu97LHbsakBfYcNzbjNh47dqxZxsXGFYDKFN5xfXfn69gvd72z3AJZ/GURAAAAANDgZhEAAAAA0OBmEQAAAADQ4GYRAAAAANBYNNPTJahG2UIgrsBNTEB2ibQu2dgtF/uR7YMTk7MnLfSzHC6ZOSb6uli7JNVMEY5sQvKkJk3Odgm8fRe4cdseP8PFwrW5ZOw4rl2ishMLf0htwrQrHODi4/qVSaDvO9ZuLMZx7IoOuHHg+hsT990x7grJZI5pl6zu1pUpPOUKDGSK4GRlir84LtHetcX+u/h9//vfb9p2797dtH3ve98be+3GdCx+JvlE/o0bNzZtUd+FQGIhE2ny+c6Ni1NPPXXstRvTLv6umNC3vvWtsdeusIab11y/4hhzcc0WiMly81Ecn9kCSa6/MdbumHF9cHP1d7/73bHXbv7InsPjPnfzTt/j+vrrr2/aNm3aNPb6lFNOSa3LjbMYj0OHDjXLZIuixG13c1a2QMikBVym4Y6TeN2QndMz48cVAHJj6umnn27a4rzixrWLa2Yb+zwHOq7IUYyr67trc2MgFj46cuRIs4wb0268xrnIfZ47/lxf49h361puoR/+sggAAAAAaHCzCAAAAABocLMIAAAAAGhwswgAAAAAaCya4ZhJ/HUJqi4hP5Ok/8gjjzTL7Nq1q2lzRW/m5ubGXu/cuXPJz5N8/2NxhSEK3PTJJTzHJNsXXnihWSZb0CMm555xxhnNMi6urgDG0aNHm7bMuqbhEsczBUlcoQOXyB+36dlnn22WiYnRUluMQpK+/e1vL9mHbHJ/LBDiCob0zRVhiOPTjTE3Pt24jnF0RRNiQRVJevLJJ5u2GOtsgrkrxhLbXKzd9kzKzUfxM7Of52IYj11XCGH//v1N2759+5q2uG+zhToyhXdc0n7f84crWDDpZ7hiOd/4xjfGXmcLrLhYx/3m+umKX5x++ulNW0bfBW7cPs+MazcOXNGveIy7uTrOC5L02GOPNW2xwI3rV7bwXjyeV3r+kKQzzzyzaYsFblwM3TnQ9S0e924Mu/Oi25fxHOsKi7jtOfvss5u2GH8Xa7ffpjHpfOHOlZmigO6a2c3zjz76aNMWr0uy50U3h8T5oe/5InLjNVPgxm2jG2MxXnHulvz5wo2neMy4+G3fvr1pO+ecc5q2eP2XLSK4GP6yCAAAAABocLMIAAAAAGhwswgAAAAAaEydszhNjkh82OeXv/zlZplPfvKTTZv77vBNN9009vp973tfs8yDDz7YtLl8hMzDUfvOWXTf8c886DmTnyi1/XX5gy4WLh/hoosuGnvtvjMdcx0kae/evU3b17/+9bHXLlfN5a1MI5NH57i4ujjGvAsXV5df5L7vHnNo4oNbJWnr1q1N25YtW5o2l1sa9R1rl/8R5wcXe5cz4PIuYp7QE0880SwTx5gkPf74401bzAdxOQPbtm1r2tx85/ZT1PdDtaOYg+zG73PPPde0PfXUU01bzPF0c+lDDz3UtLncxpgj58bvJZdc0rS5c03Mlck+YHk1uLnUjek4X7h5wc2lbr/Ffe7ya92Yzjzo2eUb9Z3b5XKO4rnMzR9uXLvrhpg35+bqhx9+uGlz+blx/27evLlZxo31888/v2mL2+QeLt53vpeLWTz/uDnL5b6581usSeHi6vLL3X6LOXlurt6xY0fTdvnllzdt55133thrl8Prcoun4Y6vGNvsdbSbQ2L83dzg5hA318Rtd9cRLtYujvF4XulcXLf+eCy548jV73DXet/5znfGXrtx/81vfrNpc8dM7NdZZ53VLOPm6iuvvLJpu/TSS8deuziQswgAAAAAmBo3iwAAAACABjeLAAAAAIAGN4sAAAAAgMaqZv7H5HuXhOuKU7gk/Vgc51Of+lSzzIUXXti0uQfiZvT9oGdX0CMW3Mgm/rp1xSRll8j/wAMPLPk+Sdq4cePYaxfDiy++uGlzD26N/XAFSlyRg2m44kQxydkt4x7YHAusSNKePXvGXrvCTXEZyRdNiEnPbgy7ogOuuEKMoytm0/e4duI4dp/pioG4QgdxfnCxvvfee5u2mJgutUngLtZx7C8k7pPMw8Wn4YphuXkgyhQNkqTdu3ePvb7nnnuaZdz84QoFxMIHbl+fdtppTZtL+I/zk5tj+ub6GwtuZB/q7GIdC3985StfaZZxsY4Pz5baec0VonDFmNz8F+cPV8ym7wI37pwX29z5ITtX79q1a+x1dly782c8nufm5ppl3FznHh6f2ca+z4uxiJXUzg9uHsvGOhZUcXF1hcfcuH7jG9849vqqq65qlnHXEq6ITzx2XRERVxxsGu48EsdPtnCTK7wSY+3Oi/fdd1/T5sZA7FcscCj5a0AXx7hNQxTJimIRxcwcs5AYe1cg6Gtf+1rT5uaPq6++eux1LFIj+QJQcQ6T2vPg9u3bm2WWi78sAgAAAAAa3CwCAAAAABrcLAIAAAAAGtwsAgAAAAAaixa4cUm9MfE9FuBwyyzUtm7d+MefffbZzTJbtmxp2g4ePNi0HThwYOz1XXfd1SxzxhlnNG2uWE5cv0tKd4UhVlqmYIXk91ssynDo0KFmGZecG+MqtfvJJd3GJGKpTSSX2vHjEp77TuR3CdTxM7Jj2CX3Hz58eOy1G6+uwIrbJ7HIkSs4lB0XMbax0IjkCxhMw+3PeNzHbZR8YRQXn9jmEsfd+HTrimPW9SE7ruOc4caci82k3NiMXNK+67sTt9vNMdkiEHH/u3Hoiv+44jWxzRVayMRmOVzMYmzdMeni75aLc7U717g2V+giHs9uTLv95gomxf5PUxgiyx03sXiU67+bU1xhohhHV5jFzSnuvBg/0xUjyY6LeIxklplWPG9JbazdZ7ptyrS5/ebGsOtXLKC3c+fOZhl3rnTrj+cCN8f3HWs3h8Txki1w447puJ2u2M++ffuaNldkMs7P7prZxTpzPsjOze4+I8PNk5l96fqemdvc+9w+c/GK2xiLOEm+2JPb/3H9Ls7LLdrEXxYBAAAAAA1uFgEAAAAADW4WAQAAAAANbhYBAAAAAI2ps3Zd4qRL6HSJ6rEYweWXX94sc+ONNzZtLjE3JoO6dbk+ZJI83fYsNzl0Ei5JP8oWN4kFXFzRHsclzz788MNjr+++++5mGZeQnEkQdgUqXLGFaWQKOri+upi5IiWZhPBNmzal1h8LeLgiTa5t48aNS/bVxTUz5pbDHXMxwdztD/c+VwglxtbF4swzz2zaYuEGSdqwYcPY63PPPbdZZuvWrU3b6aefvuS6ssfbpFxRhngMuqR6VzzKFQCI63L7Z/PmzU2bmz/OOuussdcufu74cEVvMsVO+p4/3L6M8ckWf3FjOo4ddyy7+Lh9Gde1Y8eOZpm4PyQ/Z8X4u8/Lno+yXCGQ+BnZz3Txj+P41FNPbZZx8Xfi3OPmChdrty/juIj7Uep/TnHF2o4ePTr22sXQFQ5yy8Vj5LTTTmuW2b59e9PmjvtYuMkVjHNcAbcYa3du6Htcu3kqU8gvU3RSavubmbMkX0Qsjk83ht2+dHNb5hqkzyJZcfxKbSwyBSElX2gpnlPd2HGFO911Yyyu5Yp0unngkksuadriNj399NPNMssd0/xlEQAAAADQ4GYRAAAAANDgZhEAAAAA0Fg0Z9HlrsTvR7t8PvcgzEzOovu+usthc+uP30N2D7R039t239OP63fL9C2TK+aWcd/5dt99j9vu8jDOO++81GfGcbF///5mmT179jRt7nvt8fvp7nvu7vvj03D7M+YyZB+C7L5DHsfe+eef3yzjjgeXexBzaFwendtvW7ZsadpiPoL7fn3fMg+3df1w+RQuJy4eq24+cvkBLpcuHhNuv1144YVNm8tJiHk22QeyT8rldsXYZ3IRJZ+vFnPdXI6kyw11+z/mKLqx6vJi3H6MMXT7v+8HarvjNOauuDnYtbl8tXg8u/e5PE+XUxPntXPOOadZxo1zt0/i/O3msL5NmtvlzvXuWiLGf9u2bc0y7thy4z/Gx8XVHSMuJy+ed7MP9nbHbtaBAweatjg3Z48lN6fHWLucTneN4ObquE8OHTrULJOZx6T2fJ3Ny1xp7vzgrsfc8RDHujvuXVzdOTaeF926XK5vJo7uOHXHm8s1zXDHaexDdky7vsYx7WLj6ii4fkXuGt2t/6KLLmra3DwTLfe+hr8sAgAAAAAa3CwCAAAAABrcLAIAAAAAGtwsAgAAAAAai2Z2uoTOmHDukv0zRWMclzDvCtW4tgzXr4MHDzZtmaIZfXPJzDHWbn9kH2Aai0O45Nm5ubmmLfPwZ5eU7hJ4XRz7fthtRqZggUvQd4nk7iG2MQnZFVZwRWlcLOIx4ZL23b50hXcyRXxcbKbh5oeYpJ8tHuCKOcXx6bbJzReuXzFmLtauLVN4JfPg5Glk5g+XyO8KB7jxGot1uDi4ByC7ggwxzm6sukILrhBIPD5cQRTXNg03DuPc7AoHufi7vsV1ue12BWhcwZM4j7n95uZvV7Ai9j9TVGlamQe+Z4uRuXEdC564AmuugITrV2audm3uvB6PZ3ft4sbhNAVu3L6L50V3XnfnSjdm4zGdLbCWOZbcMm5cu7b4XhcHF+tpuGMntrlx4c417hokFmpyx4grkuXGdfxMVzzFFXlzx1vcJhcH1we33zLcfUdm3Ll4uXNlLF7jrhHdNrpzXmxzc5G7lnH3TfH4OHLkSLOMK4i2GP6yCAAAAABocLMIAAAAAGhwswgAAAAAaHCzCAAAAABoLFrgxiVONitIJu0fOHBgyXW5AiiZPjiuoIRbv2tzyeSRSz6ehitQkSleky2SERO0XfKsK5rgkqDjutw+coUtMsUKXCJ538nlbmy4xOTI7XPXFmPrYuiOEbe/Y5tLznZJ725dmcIy2WIzWZlx7WLv4uMS+WNxApcI7/a3SzqP/XLjOpNM7qx0cadJi1+5ZVxxk0nj7OaBGK9pxnQcJ+7z+i6i5foW29w2Zc9lsThINtZujMWYubHqjr/MeHLFTvoukOXOxXF/uv3httOd8+J2urHv5sTMOcTt70yRJilXTGiI4nDxM1xcXXGZzJhyxTrcMe6KgcTiKa5frs1dS8R96QpF9T2u3bET+5a9JnTjPxY6crFwhWrcZ8b95s7DrpiNKwgT++/O833G2m137IOb69z73HJxG11BSLcuNw/EeT57XGWKjLplMtcto/jLIgAAAACgwc0iAAAAAKDBzSIAAAAAoMHNIgAAAACgsWiGY59FXDJFb7IFRDJcQqdrc2JfXR9cQu80XOJyTDZ2MZy0+ItLinbJs5nE+mwiuRPX5T4vu9+yMoUgMsVmJD82YmxdYQXHxT8W7HDLuH65mMVCDdMcI1nPPPPMkp/hxqKLqxtTsZBIdr+5RP4Y2+UmgC/GFTTos5iQO25ibFwhjWzBhDj3uNi4sePmrEmL0mTGuVsmU7BsOVx/o2zxEXc8x3HuCkq48eSKJsRYu8I72VhntqnvoiuZa4LsuHbiOHbrcmPYyRTuyoqfmT22puH2XRxn7jMzc6nUFuzIFqVxYza2Za9d3HFz9OjRsdfunOWuGabh+hYL67jzXbagZOSKpbh5PlPgxvXBvW/SAnp9XoO42MQ4u0I8meI8bjk3V7s5zI1pN39HLs5uG2Ocs/tnMfxlEQAAAADQ4GYRAAAAANDgZhEAAAAA0Fg0Kcd9d3jSHMJM7opbxn1/PCPbz8z35t33vbMPWNko1W8AAAKQSURBVM5y3ynOPJw089DRaWRywNx3611ejxtPcV1D5NG5uMbvb7ttyubDxXW576dncywy3HfPM/F3Dx/uOzcjk7s6aU6QlMszdLF2+yT2NZsf4vZbJu+5z/yuzEOenWzucoyF2xfZ3OW4nOu760MmXm5dmRzD5Zg0F831LZMX67bbbVNmbLrjOzsO43JD5NG5OSrKzsuZuSGbEz7pmMrMFc4QD4p3D2mPOViuVoO7FnJ5+jH+2QefZ/Jss7lcri3uX7c9fdbvkPx1bZyLM+e7hWRinTkHSu1Yd8u4uGbmh0zu5jTcuIj9OnToUOp9Lqcw7qNp6hzEfmXyLSXpyJEjTduxY8fGXrt5f7lx5i+LAAAAAIAGN4sAAAAAgAY3iwAAAACABjeLAAAAAIBGf0+dHpEt/pJJ6HSJxZMWPMkmVGce+tt3wnMmWd0lFrtEXLdcbMsW6sgUP5imEEjm8/p+0HO2IEJGZixmiz5kCpK4z3MFbjLLuQTnvosJOXEcuKR9FzOXkD+pzDjLFqPIJPe7z+uzmJDra2ZMZ8d9pjCBW5cr6hJli3u58Rrf646FvouuZAqlZAuPZbY9W7wjM7+6/eHe5+aB2A/Xh76LCbl9HuPo+u8equ22PTOnZPdbXC77sPq+x+ekMsVApjk/Z+YaNye6Ah4xZtm5x7XFOSO7v6eRmafcNWa2cFPsb7bw26TXRpmx42QL40zK9T2u3y2TOddIuYI2bv2usFyG60OmwE18LVHgBgAAAADQA24WAQAAAAANbhYBAAAAAA1uFgEAAAAAjdJ3QjoAAAAA4Mcff1kEAAAAADS4WQQAAAAANLhZBAAAAAA0uFkEAAAAADS4WQQAAAAANLhZBAAAAAA0/j+EBMbLo4+iYgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_test_with_title(most_by_correct(0, True), \"Best predicted bad CO-tips\")\n",
    "save_name = f'best_predicted_bads.png'\n",
    "plt.savefig(save_dir+save_name, bbox_inches='tight', dpi=200)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best predicted good CO-tips\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAABzCAYAAAAxOpeKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de6xd5Znf8d+CEOfi+Ep8wcfG2IAvwWCuIVGCItREkylKm4pJKibKIEUzikbpJdJUnUpVd3ZmVEWaP5pJ1Zm2U1VDVCaZTC4KCtOQdJTJBAiBBAwBczFgML6CsYNtcCAJq3+cw/Ts5/mdc56z9/aCod+PFIX1+t17v/tZ73rXWjr7eVbTtq0AAAAAAJjutFd7AAAAAACA1x5uFgEAAAAACTeLAAAAAICEm0UAAAAAQMLNIgAAAAAg4WYRAAAAAJC8YdQ3aPrNEknXtb32T8YwnmHH8D8lXSPp6bbXXjBDn0bSH0v6dUkvSLq+7bV3T/3bb0n691Nd/7DttTec+lHPH7HuDrHuDrHuDrHuBnHuDrHuDrHuDrHuDrGe2zj+srhE0u+6f2j6zcg3o0V/LunX5ujzQUnnTf3vdyT9qSQ1/WaZpJ6kd0q6QlKv6TdLT9lIR0Osu0Osu0Osu0Osu0Gcu0Osu0Osu0Osu0Os5zCOIHxO0sam3+yQ9F1JN0v6A0lHJW1u+s0HJH3rlTvlpt/8nqSFba/9TNNvNkr6L5Lersm75N9ue+1D8x1A22v/ruk36+fo9k8kfbHtta2kO5p+s6TpN6slvU/Sd9tee2RqfN/V5A770nzH0QFi3R1i3R1i3R1i3Q3i3B1i3R1i3R1i3R1iPYdx3Cz+vqQL2l67XZKafvM+SZdMte2e48v/d0mfbHvtrqbfvFPSn0i6uuk3vynp35j+j7a99tohx7lG0lPTtvdOtc3U/lpErLtDrLtDrLtDrLtBnLtDrLtDrLtDrLtDrOdwqv68emfba3fP1qHpNwslvVvSXzX95pXmBZLU9tobJd14isb2ekOsu0Osu0Osu0Osu0Gcu0Osu0Osu0Osu0OspzlVN4vPT/vvX2owN/JNU/9/mqSfvXInP90puiPfJ2nttO2JqbZ9mvwT7vT2vx3yM14NxLo7xLo7xLo7xLobxLk7xLo7xLo7xLo7xHqacdwsHpf0tln+/ZCkFU2/WS7phCar/Xy77bXHmn6zu+k3v9H22r9qJqv8XNj22ntP0R35TZI+1fSbL2syCfS5ttceaPrNLZL+Y/P/kkE/IOnfjfmzx4VYd4dYd4dYd4dYd4M4d4dYd4dYd4dYd4dYz2Hkaqhtr31W0m1Nv7m/6Td/ZP79F5I+K+lOTSaOTk/8/E1Jn2j6zb2SHtBk8ua8Nf3mS5J+KGlT02/2Nv3mE1Ptn2z6zSenuv21pMclPSrpzzRV+aidTAj9A0l3Tf3vs1NtrznEujvEujvEujvEuhvEuTvEujvEujvEujvEujC+tm3H+X4AAAAAgNeBcTxnEQAAAADwOsPNIgAAAAAg4WYRAAAAAJCcqkdnDK3pN09IuqzttYfn+bqrJH1e0oWS/nnba786Q79LJf25pDdrMln0X7W9tm36zTJJfylpvaQnJH2k7bVHp6ob/bGkX5f0gqTr21579/y/2WsPse4Ose4Gce4Ose4Ose4Ose4Ose4Gce7O6zXWnfxlsek3XdyU7pF0vaS/mKPfn0r6bUnnTf3v16baf1/S37S99jxJfzO1LUkfnNb3d6Ze/5pFrLtDrLtBnLtDrLtDrLtDrLtDrLtBnLtDrMfwl8Wm35zQZAnXD0g6qMk74meafvO3knZIeo+kLzX95ouS/qukdVMv/ddtr72tmXxuyZckrdFk2dhmmHG0vfaJqfG8PMtYV0ta1PbaO6a2vyjpn0r635osd/u+qa43aPKBlv92qv2Lba9tJd3R9JslTb9Z3fbaA8OMcxTEujvEuhvEuTvEujvEujvEujvEuhvEuTvEumYcd8tvlfTjttd+uuk3/0FST9Knpv7tjW2vvUySmn7zF5L+U9trb236zTpJt0jaMtX/1rbXfrbpN/9Y0ideeeOm3/xA/kGZv9f22v8zxFjXSNo7bXvvVJskrZwWvIOSVk57zVPmNZ1PahHrLhHrbhDn7hDr7hDr7hDr7hDrbhDn7hDrgnHcLL6syd/JStL/kvT1af/2l9P++x9J2tr0//6me1HTbxZKukrSP5Okttfe3PSbo690aHvte8cwvnlrJ3//+1p8ACWx7g6x7gZx7g6x7g6x7g6x7g6x7gZx7g6xLjgVv8OdPsDnp/33aZKubHvtz6d3nhb45BTcle+TNDFte2KqTZIOvfKn2ak/9T497TVrZ3jNq41Yd4dYd4M4d4dYd4dYd4dYd4dYd4M4d4dYG+O4WTxN0rWSvizpOkm3ztDvO5L+haQ/kqSm32xve+0OSX839bo/bPrNByUtfeUF474rnwrisabfXCnpR5I+Luk/T/3zTZJ+S9Lnpv7/m9PaP9X0my9Leqek5+b7W98xItbdIdbdIM7dIdbdIdbdIdbdIdbdIM7dIdYF46iG+rykK5p+c7+kqyV9doZ+/1LSZU2/ua/pNzslfXKqvS/pqqbfPKDJP+XuGWYQTb+5vOk3eyX9hqT/NvV+r/zbjmldf1fS/5D0qKTHNJkYKk0G+P1Nv9mlyT83f26q/a8lPT7V/8+mXv9qIdbdIdbdIM7dIdbdIdbdIdbdIdbdIM7dIdaV8bXtaD9rbfrNibbXLhzpTVBCrLtDrLtBnLtDrLtDrLtDrLtDrLtBnLtDrGs6ec4iAAAAAOAflpH/sggAAAAAeP3hL4sAAAAAgISbRQAAAABAMuujM77xjW+k36i+/PLLA9u//OUv0+teeuml1Pbiiy+mtvhep52W713POOOM1LZo0aLU9pa3vGVge/ny5anPkiVLUttb3/rW1PamN71p1u2Z2pqmmfmBK3P4zGc+k2J9+umnD2yvXLkyvW5iYiK1rVixIrXF7xnjJflYv+ENeYrE/eT6uDb3/tEvfvGL1BbniSQtWrRo6Fh//vOfT7GO442xl/w+X7BgwZz93Pd2bW6fxLbqe7ljKX4n18fFf/PmzUPH+utf//qcsXZzxcXa9XvjG9847z6SP+5jP/e66lyP3Bx26+TKlSuHivUXvvCFFOe4f92croqvdfvHxdQdHzGubt67tXrhwlyDIL6Xm9NurEuWLBl6Tn/rW98a6rzo5oATv5OLYXWex9i+7W35kV8urpU1xY3LtZ1xxhlDx/qGG25IsX7zm98867YkufQaF//Kuu9i4U718b2qr3NzJa7DL7zwwpx9JOnDH/7w0LFevXp1CtqxY8cGtteuXRu7aNWqVanNXe8dPnx4YNsdl5dddllqu/rqq1Pbtm3bBraXLl2a+rj19ec//3lqi7Gtvu7973//0LH+yU9+Muca4uZr9TwS29zr3FpZUT1fu3U9niMq505JWr9+/VCxvuWWW4bKs3Px+tWvfpXa4jHo9oVbU1xb3B9uPXdxXrx48Zz9quvahg0bZowzf1kEAAAAACTcLAIAAAAAEm4WAQAAAAAJN4sAAAAAgGTW7FKXoBq5hE6XOOsSu2MSerUAwPPPP5/aYqKsS0h2Yx1nwYpRCkhceeWVqS3GzCW8uiRVN7YYM5eAXk02jp9ZSWSuvpcrVjBulcTuSgLyTGLSs0uMrrxuprbKuCqFcNx7u3kxisox7cZfTeSP/dwxMspnRpWCT1IuWuHe261Rw3JjqMaiorK2nTx5svRelXWmuhbF710tRjYKt9+GneeVueMKoLh968ZQqbnmXueKJsTiOG6tHuUc6LjzSPyMaoGvSlu1yJtb0+P1THXddzEbdl0bhbsGueeeewa2n3nmmdTn+PHjqc0VUopFq84999zUJxaukaR169altjgvKgXdJB/HuJ+q7zUKVwgqzh93vnNrgVvf4njdXKycT6U8/93nVednfP9qoa5TqXqtUVn3XZyr5/nK93brk7uOi/vfvfd85zR/WQQAAAAAJNwsAgAAAAASbhYBAAAAAMmsiSGV/LFqzp/7zWx8GKrLeanma8R+7rfD7r1cTkfl4dzj/g37ihUr5uzjxu9+W+1yCIaNjxPzEZYvX576uLhWcv6qD48dJf7ud9+VPBj3mS7Hr5KrcuLEidTm8hgqD9WuPJTVjcv91v25557Lgx1B5cH2bp9X52Kc//Eh0jO91wwPah/YdvO68n2kWn7oOHMW3Vo97PFWyW9xeePuO7rjaNmyZXOO072/O/7ifnT7Z9w5MJU1qpKLKNVy66sP53bzMK4f7uHlLq4uDz2en92D791x5eZAlYtPfD/3/u54qMwDtwa7tsqx5c6B1VjE7x2vlaRaPvt8XHLJJaktxvGpp55Kfdw+crmmMfdw48aNqc/b3/721OaucWLupJt3w143nOq1WpIWLVqU2uL64OaPO8bd2OKx6s6L7nrbxSzuS7f2LFy4cM7XSfkc69aZar56Rfw8KZ/f3DVctT5FjL279q7OnXhtV821r5wXXZznm8vPXxYBAAAAAAk3iwAAAACAhJtFAAAAAEDCzSIAAAAAIJk1k7SSQO2Sj12bS7A9evTowLZ74Kt7nUvojAViVq9enfq4xFyX5Bn7VYttjFJ0xSXkx0RcF1eXnOuSbGOsn3766Tn7SD7Rd9WqVQPb69evT30qD6mWcnEWtz8qD5aeD7efYuEB971dorIrPBCL1/zsZz9LfapJ1bEYyMTEROrjHlrsijLE7+i+z5EjR1LbuMW54b63m+uu6Mmzzz47sO3msIu1S6KPc9HF+pxzzkltrihDnMduHXvsscdS26ZNm1JbRTUhv6JSUGLfvn2pj1t3XEGPNWvWDGy79cONwRUocQWfTrVKoZTq+ufOLZXzoltT3JyO64dbK5xK0SbHfZ9RCty480EsnOHOndUie7GfWyviGiPVijm5ohKusEnlfOTW80ohtflwBWdika9qAQ83tvid3L5189p9ZpwDcZ5LvkBZZb1w+2PcBQ0rBapcfFwBOnfOjuvzwYMHS69z+y3O47Vr16Y+7rzo5n+cK+68Nd/CK7Nx36dy/eG4eRgLPu3fvz/1cXPazad4/RGvsyUfexevOM9dHOa7fvCXRQAAAABAws0iAAAAACDhZhEAAAAAkHCzCAAAAABIZi1w45LEK1yS+8mTJ1Pbrl27BrYffPDB1Mcll7tE9Vg0wRWKcK9zSdBx/C6ZfdjYzKRt2znH4eLqin7s2bMntT3wwAMD2w8//HDq44qDuCTlmGTrknrde23bti21xYT/cRezcVyxgMgloMfCNZJ06NCh1LZ79+6B7SeeeCL1cYnqbk6dddZZA9ubN29OfSrFBCRp6dKlA9tuXrv9NopKMRBXzMTFx82zOI9d0ZjDhw+XxhWLf2zZsiX1cQUwnLiuuNe5wiXjVCnK4IpTxKR9Ka8fDz30UOrjivi45PuVK1cObLu1+uKLL05trpBJTOSPhS8kP7/GWURBykUSXNEEVwzrwIEDqe2RRx4Z2HaxdnPafWYsvuTmtJsDleIsbo0cZ6Elya+Jlc9w50p3DMbzpzt3unXHXc8sXLhwYNsVjHGF99ycjaqFBEexZMmS1BaPr2qRN1fUI54r3Trj5rW7NornsnPPPTf1cfF3RUPiXD/VRVekPFekPGer+9xdI8dr63vvvTf12bt3b2pz11/x2trtb3ecujUk7jfXx7WNU9y/7jrIzd/HH388te3cuXNg263V7jzvzs1xra4Wu3PrR4zhOOYvf1kEAAAAACTcLAIAAAAAEm4WAQAAAAAJN4sAAAAAgGTeVVoqBUhccQpXvOb2228f2P7pT3+a+rjk05h0LeXkaVdswRW1WLZsWWqLyccuedcVEKkUlJiJSxqOSfQuri4h/+67705tt95668C2K7riEotXrFiR2mKS+5EjR1IfN9ZK4vjExETq42Ltil1UucIMkSsc5L6nKxR0zz33DGy7WDtnnnlmaotJ766YjTtG3HfcunXrnH2OHz8+5zhHFT/XJe27BHMX6x/84AcD267AjTt+K7F2cXXccRPntTu+3Xwalktyj2NwxSlccYR9+/altljgxs1pd5y6ohlx37rPc4XHYmEc1+bWq3EXp6gUN3OxdnPaxfHOO+8c2I5FFGYSi0dItbXOcefKOMcqhVlG5Y6tuPa7/eGuU9w+ifF31yDuGHn66adTW1ybXQGRSy+9NLWdd955qa0S2+r6VOWOX9cWubXNxSxeA95///2pj7tuqBTJcsXn3P521q9fP7Dt5ty4Y+1iFs8/7hrEzTtXvObb3/72wLaLtbsec+tnPD+7feS+j1tDXIGnaJyFs9w1edyXbp647+jOUzt27BjYdtcoTixmI+V7FldQx3HHaLyvcefT+eIviwAAAACAhJtFAAAAAEDCzSIAAAAAIJk1+cL9djj+BtjlG7kHq7qcxZgf4H4XvmHDhtQWHw4s5d/7uge+ut92uzyP2LZ48eLUZ9zc76Zjm8snc3kRMb9IyrlcLu/ziiuuSG3xofBS/t28ezC9mzsujvG31G5c7nWj5Cy637/Huece4Ozm9ZNPPpnaYh6M+035tm3bUtvZZ5+d2mLegstzc3l6Llcs5gu4BwMPm+M0ExfrSm5G9fh99NFHB7ZdjsJFF12U2tauXZva4vF19OjR1MflJLj5GePv8m7GGWuXHxJVcxZdHl3M13DfZ/v27alt3bp1qS3m7rk8nPvuuy+1uTyumGPj1qthc7HmI847N+/dPN+9e3dqi8fzggULUh+3frg5HdcLt1a7c7Ob0zG2Lt/ffW83/iqX51nNRYvc2hm/u7uecdcgLq/te9/73sC2O45cLqV7rxhr951dntgoKuuR6+P2UXwovJRzcffv35/6uHnn2mI83HHkzgXu+iLO40oO+qhcHGOb27+VfHIp57u5c/0FF1yQ2lwuXXyovMsPdddBq1atSm3xGsTF2l27uPFXuHuKmFvs6kBU53Q8xt33cedFd56K5+IYd8lfF61Zsya1xfG79WO+c5q/LAIAAAAAEm4WAQAAAAAJN4sAAAAAgISbRQAAAABAMmuBG5esHttisQrJF4ZwifXxYbrugbUf/ehHU5t7OPMtt9wysP3DH/4w9XFJ0K4owKZNm+bs4xJnR0nkb9t2zj4uEdc9PNQlxsaxuaIfH/nIR1LbJZdcktpuvfXWge2bbrqpNAb3kNFYPCA+IFfySenjFpPLXYEbl/Ts5nW0cePG1PahD30otV111VWpLSav33zzzamPKxDiigfEAg8uGdu1jcKtDzHZuvrAejenYsGIiy++OPX5+Mc/ntre9a53pbbbbrttYPub3/xm6nPw4MHU5uZA/I4uad+1DcutDZVE/uoDteNr4xop+fWjEuevfvWrqY8r3LVnz57UFue+G7vjirNUVYpruWI2J0+eTG3uXBmLFZ1//vmpz8c+9rHU9u53vzu13XHHHQPbX/nKV1IfV7jLFVKIc98Vfhn3Wu3iU3lgvSsg4uZPPH9u3bo19bn22mtTmyso9elPf3pg250XXTEhF8dYZM/Nuco1w7hVi4G4WMf12xVTec973pPaXOG3WFDFzdcDBw6kNtcvFuFy46rMuflw8zOeK93a7I5V9z3jtbUriHX99denNnf+jIWb3HnRjcEVeNqyZcvAtotDHPso3D1M5K7l3XnEXX/EdcBdV1933XWpzfW7/fbbB7bjPY1Uvy6K/dxxS4EbAAAAAMDIuFkEAAAAACTcLAIAAAAAEm4WAQAAAADJrJmkZ5xxxpxvEAtMzEcsMnDOOeekPldccUVqc8m6MVE9FgaRfCEKl0AfE4tdkY5XI7ncJeK6NpfMGhO0XRK3K/rgigmtWbNmYHvx4sWpT6XAipT3WywMIvn4j8IlVbuYVV7n2mIxoRUrVqQ+q1evTm0TExOpLc5PF2uX9O7iGOeKK2bjxjoKF5/Ixd6N/6WXXkptCxcuHNg+66yzUh8Xa1c0IRZVWb58eerjijS5+Md57cbl3n9YrrhWTL6vJrm7tjhWV5QjrgtSLh4hSfv27RvYdnPOFc04fvz4nP1cYZlxF12pFE1wRRrcseDOI3FOu7i6Oe3a4vrt1g9XIMsV14jHpDseXWGwUVSKCVX2h1Q7f7r3coWoVq1aldriMe6OyWrMYjEkdw4cd6wrcXTrsiuy547fuK64IlnXXHNNanvve9+b2mIxkK997WupT1xnJF+gLBYzccfpuAvcuOMrrs/V6xS3hsRYuzXWFW6MBWikXLwmFjiU/DnQrcVx/rvjwbXNtxjLbCqF35zTTz89tcVzi7uudmvFeeedl9pioZof/ehHc/aR/DEZuYJc88VfFgEAAAAACTeLAAAAAICEm0UAAAAAQMLNIgAAAAAgmbXAjROToF0StysosGjRotQWk6BdEvfOnTtTm0vy3L1795zvVSkCIeXv6BLJXVLpKIUU3NiiWPhA8kVpXFtMlnbjf+ihh1KbS15/6qmnBrZjMr7kk4Fdknhsc4nMldiMKhZzcsWdqmOL390lqsf5KvnE7pikf+LEidK4XCGLWNDG9Rl3rCsFbtxcccV3XLGLWCjAFQ5w89rN2ccee2xg2yXou1i7tS3Oaxfr9evXp7ZhuYIqMa5urXPfxx2n8XhwsXFxdoUu9uzZM+d7Oa4AR/xOLg6usMko3GfEQm/V86IrnhILbrjzz4MPPpjaXKyfeOKJgW13fLi4uu8Y54ArDOHefxRubMMWzXL7JMb2vvvuS31uvPHG0vvff//9A9uuOJwrdOXGFb+jKyQ4jqIVs31mlSuw4vZbnP9Lly5Nfdza49aoeI3jXue+j1sL4jrpPq9aRKnKjSN+hjsG3fd014Xxte64vOeee1Kbu7aORd3ctWO1UFplvR7nvHZzoLK/q+fFuOa6/RqvKyR/rbdr166BbRdnNwZ3vR/PK26NqRQwnY6/LAIAAAAAEm4WAQAAAAAJN4sAAAAAgGTWnMXK77vd74vPPPPM1OZ+v//kk08ObLvf9saHr0r+t7bx977xYeaSfwi2e4hm/L2y++2w+w27e68qlwMR8y5dbpf73b/Li4gPXo7xkmq5AZJ05MiRge340FbJ/xbdPRg27hP32+px5xxV8g/cOFys3QNXY16hi88DDzyQ2ly/Y8eODWy7B5O7fLiJiYk529xxWn1AbVUlD8bNOzc2N69jfpfLBXW5YjFvTsr5By7WLj/ErW1xrrvv4/Iyh+VytKPqA8fdQ+Bj7mxcuyWfA+OOmbjPDh06lPq43Aw3rhhnt+5UcyKrXL5JXC/cedGdf9yDsWNOuIu1y9+N+YlSXqtdDrqL9erVq1Nb3JfuPOxiMwq3flTWb5fj575nnD+PPvpo6vOd73wntVXyNc8///zUxz2M282LeK53n9fFeTGOw8XQrSEujzvOPbdmuTnsrr9iPzev3frqrtHiNU41h20UlXntji93Hlm3bl1qi9cSbg258847U5s7f8bzoou1i4+7Norxd/NknPmh7hip1Kdw88Sd5+N1tcv5fPjhh1Pb/v37U1t8rTtvuWu9SpwrdQjmwl8WAQAAAAAJN4sAAAAAgISbRQAAAABAws0iAAAAACCZtcCNS8KNifWuoIBLjr/wwgtTW0zodAVuvv/976c295nxIduuAMA73vGO1OYSzmMyqEvEdknXo6gUBnAJ267ox5YtW1JbLHQQH7QqSXfddVdqcwnIcV64hGSX3L9169bUFotWuH077li7QkFRNdZuXkexYIUk7dy5M7W5YiBxHC7BefPmzalt+/btqW3Dhg0D2y5ZftxFE9waEos1uH3uEszd8RsLALlYu0R+9/6xKImbA5s2bUptF110UWo7++yz53yvcXJrVIxrtQiBm0+xKM0jjzyS+uzYsSO1ucT6yv5367IbVywO4uZvF0Wb4ue6NdEVC3Pr5OHDhwe248PeJR9rt1ZHLtYbN25Mbe4cEo8ZF4d4Hh6VKygVi9dUzw+u0FVcU1wMXTGKF198MbXFY8kdW65Ik7tWqcyncXNFmeLnunXMfc+4/kl5DXHFQFzht1hcS8rFx9z65/alG2ss0OPmiVvHRuGu9yoFDd21tTvXx6I0Lq6uIJkrVhTH4QqjuHXsggsuSG0x/m5eu+PZzc0KN9bY5r7z+vXrU9vll1+e2uIa6Ao0ubXafWYcl5u/7rzorqtj4TQ3pylwAwAAAAAYGTeLAAAAAICEm0UAAAAAQMLNIgAAAAAgmbXAzYIFC1JbTHCuFqe49NJL84eH1/74xz9OfVzRmxMnTqS2mPjrkj5dIYpYIEHKBRFcwq1LqB6Fe7+Y/OsSUt343XeP7+Ved+DAgdTmErFjcq5LunWxduOKSbzu81whhVFUCl645HJXgMYVgojJxK6YUEz2l3wBkhUrVsz5edu2bUttLtbxuHTJ5fNNep6LKwQRP9etIS4B3BXiiK9160UsGOLGIOXkezeHL7vsstTmCu/E8bvPc/t7WO79K0Ux3Bhccn88Ztz+ccn9bv/HwkquaJArHOXGFcfvju1xF7hx3ylyc9oVMjnnnHNSWxyvK06we/fu1ObOU7FAzLnnnpv6VAqPSXkddufhccfaxSwWuHFrVtu2qc3N2ZUrVw5suyJm7nu6c1IchyuK4q6pKsep+7xKUbz5cOOI50FXaMQVjXHFTWLRFXderBZdibF2x5srsuOuVdatWzew7QqLjLvAkDtW47W1KybkxuHWzyjOc8kXg3NrWyyqF+Ml+fXarW1RLLwo+XNSpXiX4wqexRi6Oe3uYdw6GY8PN1cPHjw45zjda935zu3rWMxGyt/xhRdemLOP5M81f99/xn8BAAAAAPx/i5tFAAAAAEDCzSIAAAAAIOFmEQAAAACQzFrgxiWaxmRQl1jsXueSdWMC+MTEROqzf//+1FZJDnYJqsuWLSuNKyZ+uuIz4y664t6vUgjEFQBwCedxn7gk5aNHj6Y2lyAck8tjcaGZxuCSlGNsK8UjRuWSfSNXNMG1xeRvKRcxcPFx+9sVP4jv744RV4zCzfU4n8Y9h51KwRFXjMLF2hWfiEnhrgiO298uqT0WE3KJ4y7WLqk9HquxKIfrM5Tl1dMAAAQzSURBVAq3L906GVWT++M6445vV6zAFeGI64ArtuXGEIu1OG6+uTVsFC7Wcd2qFkBz3ykWoakWMXOfGeeYey9XuMvNi/i9T548WRrDKNxnxLG5YmTueHNiP3dMumuEyvu793LxcetffK1bD8dddMXFMXKxcGNzRb/i+8dCV5K0d+/e1OaO33jcuHV/+/btpXG5c2VlDKMUg3PzIK6Vbp1xc8qtxTE+7nrPrdfuXBk/s3JfIPkCOocOHRrYdmuP2x/u3F/hzoHxuHHz3u1bd06KsXDXDLGw00zjip/pis24a+jK+7tiW+66hQI3AAAAAIB54WYRAAAAAJBwswgAAAAASOadNFPJc3K/53e5dfE36+732O53wpUHt7vf81dzhOJ7ue9c+X3/fLjffFdyEtx3cnl08bfI7iGtLl/Q/bY+jsvtW9fmYlaJ9bjzYFxe07D70/22PeZ1uJzFav5pzD1w+9a1uXFVjt1xz2uXcxS5ue/G4WIWf3fvcrJcnk0lr9rlB7jf9Ffyu6p5c8PmwVRyINx7V4+tGBuXA+PyaSvcOudyLNycqOQ4jzM3VKo94Nh9p2qOWZxP7hzocn0qa7WLRWX+SrVjedx5dG6tjuN143fXIJW5Xl3/3LyLMXNjqJ7nx3k9M4rKGlLNL49riHvo+LPPPpva3NoW12+Xt1fNL4+ef/751Fb93lVuDYnzs1r7o3JedPUQ3LrixvXMM88MbLu8UpefeOLEidQWjwk3Bpf3d/nll6e2Crd+xGOrej3ozkmxrRpTN8eOHz8+sB3jLvkaLvF1jjs+NmzYkNpivvx0/GURAAAAAJBwswgAAAAASLhZBAAAAAAk3CwCAAAAAJJZM6Sfe+651BaLy7iEW5fE7ZLLY+Jy9SGklQdQVx56OlNbpejKuJPLXQLysMUCXPwjF5/qA4NjkvIoxRxiHF1cx100YdwFc6J4TLhEdbe/Xb/Y5pLqXYK2KwoQi6y4Y9cZJf6uEERMHnfHl/tMN68rsa4+RDj2c/vIJb67NSTOsXEXSJjr8xxXZMfFufKQ4nEe89XXuThHlbVvVJVYV88ZlThWClNJte9eOXfONK74/uNel6vieufGUV3b4r6sxrry3d17Vdviedet8e7cPIrKcV8tAFRZX10xEHe+cMd9HEf1OtTFMRYIceuye6/ZHmA+F7eGVI5ft4ZX1ppqwUH3gPcnn3xyYPuuu+5KfXbt2pXajh07ltrid4xFLiVp48aNqW1YlflaPebdvIjvXymoI/kCNwcPHhzY3rlzZ+rz+OOPpzZX4CYeD67Yk9vX11xzTWp7BX9ZBAAAAAAk3CwCAAAAABJuFgEAAAAACTeLAAAAAICkGXeSNAAAAADgHz7+sggAAAAASLhZBAAAAAAk3CwCAAAAABJuFgEAAAAACTeLAAAAAICEm0UAAAAAQPJ/ARYVjJcbSCMWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_test_with_title(most_by_correct(1, True), \"Best predicted good CO-tips\")\n",
    "save_name = f'best_predicted_goods.png'\n",
    "plt.savefig(save_dir+save_name, bbox_inches='tight', dpi=200)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Worst predicted bad CO-tips\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAABzCAYAAAAxOpeKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2de7Cd1Xnen8XtIC5CVyQhgS4IAQKKp4hbPSS26TQ0oXFS29PYSUo67sWNA+50cMdu6zrjTlsSu2C7GeLUTm1qO/WFxKlDPa4JGNvEBYITKZbERUIIXQAJXY3A5iJW/9jfQXu/6znnrLP3Pvtc/PvNnNH+1l7722u9613v+pb297xfyjkLAAAAAAAAoJ3jJrsBAAAAAAAAMPVgswgAAAAAAAAFbBYBAAAAAACggM0iAAAAAAAAFLBZBAAAAAAAgAI2iwAAAAAAAFDQ+2YxpTlK6Tf70JZe2nCdUnpMKW1VSh8Yoc6QUvpyU+dBpbSi7b0PNuWPKaWfG0yjuwBbDw5sPTiw9eDA1oMBOw8ObD04sPXgwNaDA1uPTc65tz9pRZY2jvDeCT2ff+zvPz5LT2RpVZZOytKGLK019X4zS59qXv9Klr7cvF7bfGYoSyubcx0/4e3G1tgaW2NrbD0zbY2dsTW2xtbYGlvPEFv34zbUWySdq5TWK6WPKqU3KaXvKaWvS9qslFYopY2v107pZqX0283rc5XSN5XSD5rPXNDF918haaty3qacX5b0JUlvNfXeKumO5vWdkq5VSqkp/5Jyfkk5Pylpa3POqQi2HhzYenBg68GBrQcDdh4c2HpwYOvBga0HB7YegxP6cI4PSLpYOb9BkpTSmyT97absSbX/TFry3yW9RzlvUUpXSrpd0luU0q9Ker+pv1U5vz2ULZW0s+14l6QrzWeP1cv5VaV0WNL8pvyB8Pmlo7R5MsHWgwNbDw5sPTiw9WDAzoMDWw8ObD04sPXgwNZj0I/NouOhZnc7MimdJunvSPqqUhouHZIk5fxFSV+coLbNNLD14MDWgwNbDw5sPRiw8+DA1oMDWw8ObD04sHUbE7VZfKHt9avqTKRzcvPvcZIOvb6Tb2d8O/Ldks5uO17WlEWG6+1SSidIOkPS/nF8fqqCrQcHth4c2HpwYOvBgJ0HB7YeHNh6cGDrwYGt2+mDMHN+lp5qO35Tlu5qOz4xS/uaekNZeiBLv9289/0svaN5nbJ0aRfff0KWtjWizmFh6EWm3nuDMPQrzeuLgjB02xQW4WJrbI2tsTW2nuq2xs7YGltja2yNrWeIrXtPcJPzfkl/oZQ2KqWPmvdfkfQRSQ9JulvSo23v/qqkdyulDZI2yQs6x/r+VyX9lqT/K+kRSV9RzpskSSl9RCn9YlPzDyXNV0pbJf1rte5RVlP3K5I2S/qmpPcq56PjbscgwNaDA1sPDmw9OLD1YMDOgwNbDw5sPTiw9eDA1mOScs79PB8AAAAAAADMAPrx6AwAAAAAAACYYbBZBAAAAAAAgAI2iwAAAAAAAFAw9TaLKW1XSgu6+NyQUvqyUtqqlB7USA/RTOl9jYh1k1L6V23llyql/6eUfqiU/kwpzW7KT1RKdzTljyilD3bTrSnJ1LP1CqX0Y6W0vvn7VBe9mppMnq3nKaW7ldKW5t+5Tfn72+y8USkdVUrzuunalGLi7TxHKd2plB5t4sHVbe/d2JRvUkq/25Rd0WbnDUrpl7vp1pRk4m29vYkR65XSw23lxI/6z02UrecrpW8rpSNK6fe66dKUZaJt3ap7vFL6a6V0V1vZ55TSk23+O/yA8LlK6WtK6W+U0kNK6eJxt22qMlm2PvbeJ5XSkbbj5UrpnsbW9ymlZeNu21Rl6tn6PW0x536ltHbcbZuKTF78WNl8bmtznpOa8r7YeTCbxdbzQCaad0s6qJxXS7pN0u+Ydlws6Z9JukLSpZKuV0qrm3c/I+kDyvkSSV/TseejvEPSUFN+maR/MeogTjbT29aS9IRyfkPz954J7EPvTA9bf0DSPcr5PEn36Fj2rI++bmfpg5K+o5wPTHBfumOq2LnFJyR9UzlfoJatH2na+Ga1sqBdqpwvkvSxpv5GSesaO18n6Q8G1J/umFq2lqQ3N366rq2M+FHPRNn6J5I+JOnmCWhz/5l6tn6fhmNHJ+9v89/1Tdm/lbReOf8tSf9YrRg0dZkutk5pnaS5ofRjkv5nY+uPSPovfWntRDG9bf1HyvmSZm38XUm39qe5E8D0sPPvSLqt+fzB5nxSn+zc+2ax9T+Lt6n1v+n3KKWFTfl9Sunjzf9Svk8pLVRKf6yU/rL5e2NTb75S+lbz+c9ISl225K2S7mhe3ynpWqUUz3WhpAeV84tNqtrvSPqHzXtrJH23eX23pLc1r7OkUxtnmSXpZUk/6rKNvTHzbT11mDm2bv/8HZJ+yXzHOyX9ry7b1xvTyc4pnSHpZ9RKXy3l/LJyPtS8+y8l3aKcX2re29v8OzwmUutBvpOXfno62Xp0iB/1TIytc35BOd+v1qZxcplutm79WvULam3Ea1gr6V5JUs6PSlqhlBZ12cbemCm2Tul4SR+V9G/CJ47ZWvq2unkMQr+Y6bbOuf06+lRN1to4E+zcqveW5nNS+7Ven+zcj18WT5X0cPO/6d+R9OG2905SzuuU839V63/DblPOl6u14Ax39MOS7m8+/zVJ57z+6ZS+p2O3ZLT//V3TjqWSdkoafmbJYUnzQ52Nkq5pBvcUST8v6ezmvfbno7yjrfxOSS9IekbSDkkfm8RfYGa6rSVppVo/r39HKV1TaZeJYKbYepFyfqZ5/aykzouMVv3rJP1xjVEmgOlk55WSnpP02cZHP6OUTm3eW6PWGDzY+O7lbe24UiltkvRDSe9p2zwOmulka6m1qH1LKf1AKf3ztnLix9Sw9VRhutn642pdOL9m3vtPat0CeZtSGmrKNmj4P/9SukLSckmTdXvkTLH1b0n6etvaOMwxW0u/LOl0peTOOwhmuq2llN6rlJ5Q6xevm0Y2xYQyE+w8X9KhtmuLXc35htvRu51zzr39SUezdELzelWW1jev78vSz7bV25ul9W1/u7N0WvN6VVu9A1la0EU7NmZpWdvxE/Y80ruz9IMsfTdLv5+ljzflF2TpW817H87S/qb8jVn6YpZOzNKZWXqso72D/Jv5th7K0vzm9WVZ2pml2di6J1sfCvUOhuN/lKU/mxQbTzc7S+uy9GqWrmyOP5Gl/9j2+f+WpZSlK7L0ZFbrObZtn78wSw9l6WRsXeXTS5t/z8zShiz9THNM/JhsWx/73G9k6fcmxcbT0dbS9Vm6vXn9pizd1fbekiZ+DGXpjiz9h6Z8dpY+27Tz81n6yyy9AVt3aWvprCzd39aPI22fOStLf5Klv86t+L4rS3Ow9QTYuvPz78rSHdi5azsvyNLWtnpnZ2ljP+08Effh5rbXL7S9Pk7SVcq587aV0e6ISel7kk4379ysnP88lO1W6389d6l1y+gZkvaXrct/qOHbyFL6z2rtwIdv7/h7TfkatX7mlaR3qaVRekXSXqX0F5LWSdo2csMHxsyydesWvuHb+H7Q/E/IGkkPa/KZnraW9iilJcr5GaW0RNLe8Mlf0WTdguqZynbeJWmXcn6wOb5TwxrQ1nt/opyzpIeU0muSFqj1S2TTs/yIWgL/i4VP1/j07ubfvUrpa2ppcr9L/JgCtp7aTGVbv1HSLyqln1frtvTZSukLyvnXdOyXl5eU0mc1rAdt3Ub2T5r2JElPampcf0jT0dat9W61pK1Ne05RSluV82rl/LSO/Yp7mqS36ZjUYLKZWbbu5EuSfn/kBg+U6WjnX5c0RymdoNavi8ua80W6tnM/bkM9TtLbm9fvknT/CPW+JenG14+GM3219BDvasr+vtpFsDlfo2Ni7/a/aGRJ+rqkG5rXb5d0b3Ph1klKZzb/nqNWUPijUH6cpH8vaTiT3g617gWWWrecXSXp0RH6ONHMbFu37gk/vnm9StJ5mrxFcWbYuvPzN0j6322fOUPSz3aUDZ7pY+ecn5W0Uymd35RcK2lz8/pPJb25accaSSdJ2qdWhrITmvLlki6QtH2EPk4008fWKZ2qlE5//XVrw7KxOSZ+TLatpxbTx9Y5f1A5L1POK9T6j7p7lfOvNd+9pPk3qaU3Gh6DORrObCj9U7U28ZOTN2Em2Drn/6OcFyvnFc17L76+eUlpQePrUivx2/8YzRgTzEy39XltZ/gFSVtGMsQEMxPsnNXS2A7349i1Xr/s3IefcI9k6dbmJ9R7s7Sw7SfcdW31FmTpy1n6myxtztKnmvL5uXWby6YsfTpLT3X5E+7JWfpqlrbm1q1eq9p+Bv9GW73vNd+/IUvXtpW/L0uPN3+35OFbyFo/M3+1ad/mLL1/Un4q/+mw9duatq3P0l9l6R9g655tPT9L92RpS5b+PEvz2t77jSx9adJsPD3t/IYsPdy040+zNLcpPylLX2j68VdZektT/uvBp38JW1fYunU70Ibmb1OW/l3b54kfk23r1nvbc+uWqyO5dbveWmxdEUOO1Y+3od6bpR82/fhClk5ryq9u7P9Ybt0iORe/7tHWsU/HXr89t9bKx7P0mSwNYesJs/Un8rF4/e0sXYSde4ofq5rPbW3OM9RPO6ecc1ebzNdJ6YhyPq23k0AV2HpwYOvBgJ0HB7YeHNh6cGDrwYGtBwe2HgzYuYrBPGcRAAAAAAAAphW9/7IIAAAAAAAAMw5+WQQAAAAAAIACNosAAAAAAABQMOpzFm+//fbiHtVTTjml4/j4448vT3pCedr4OUkaGhrqOH7ttdeKOq+88kpRdtxx5R73xBNPHPNcR44cKcpefPHFMb/TtcF97uabbx7lgSuj841vfKOw9UknndRx7GztbiM+evRoUZbCs2CcDZ3Nas7vPvfjH/+4KHP2r7G1K7vpppu6tvWtt97a1b3Xrp/dfq5bWzsm+lbyD33oQ13b+qabbioad/LJJ3ccu3jh/PPUU08tyuK5HO78MV64ei+//HJR5/Dhw0WZ8/VXX3111OORym699daubP34448Xdp43b17HcYwnI+H6Hfv4wgsvFHUOHSofR+b6WOPTrg0vvfTSmOd33+fixzvf+c6uffquu+4qbB37vmfPnuJzbs2oWSvPOOOMos7ixYuLsuXLlxdlc+fO7Th2PhDXBsnHlBi/f/Sj8ukNbowuu+yyrm39uc99bsx1cdasWcXn3PWGqxfjjIsnp5/uHpNWEvvuYli85nGfk6QDBw50HLv55ubR9ddf37Wt77777sLWsb3Ori4uu/gay2qvHV29mrXY+bCzdYxtNXUk6aqrrura1p///Oe78mtnH+ezsZ6b9+5zNWuEs72Lu67eT37yk1GPJR/7r7vuuq5sfcstt4zp087ONdcVUtlW12c3Zq6spo5rl4szNdfVLn7ccMMNI9qZXxYBAAAAAACggM0iAAAAAAAAFLBZBAAAAAAAgAI2iwAAAAAAAFAwtsoyEIWStUk/akXDNThBZxSA1yb9cCLS2C6XmMAJc3th9uzZRVkUhNcmInBi1ihArhWS1+CEsrXi6eeff77j2PlJt+0aCSe+j37s/Lq2HXFMapMJ1cylmuQgUr2vdNOG8eD6Hn2v22QUrsyNbS1xjrjvc8kuTjvttKKsRvjeT1u75DJxvs2ZM6eo4xJuOGJ8dbG7to8xdrp45XDjEftYK+TvhQULFozZDpeUpiZxkFTGmYULFxZ1Fi1aVJTVJG1y7XJxzbUrxpSYPEfqf/xw60jsZ23SFUf0KZeowyWXcWtZTeypTQQSY4rzfTfne2HJkiVFWYx3bjxcDHH2qfEN54uurCYRi4sFbg2M8ciNt7sG7AW3vsWEZG5dqSXOezduzhddjIq+58bbjZGzfxw31y7Xhm6pSYbpcOPtzhXH0dnBrT/ONtGubvzddbubV7GtLh6O97qaXxYBAAAAAACggM0iAAAAAAAAFLBZBAAAAAAAgIJRNYs1D/t09/G6e2HdA3yjXs1Rq0GK9wXXPBRW8vcOx7Y6HV2/qbl/2N377O7vdmWxn07T5sa79iGgEdcfd697bJdru7u3vhdc32N7XftrdbARd0+5a0PNZ2vvM6/RafVbX+RweqKodXL307u2OfvX2LFWMxDjQ+1DcZ3P1jx0t5/237t3b1EWNTBOJ+Nwmp0Yq2u1Vy52xnq1D5+u0UQ6f6jVr9Xi5mDUELq2Hjx4sCir0fQ6WzjNfM2DpI8cOVKU1Wo6Y5+crtH5jtNJ1lLzwOyauSZ5nVv0Y9f+w4cPV7UrxrparbprV7eavF7ods2rjQU1Dwp33+fGN8bq2uuUmra6GDIILX8sq4kNIxH7uW/fvqKOiyE13+l8s0ZP7spcG/qZD2TlypVFWc01VI0OUCpjp2u7i69uL1ITv10bXC6CqDWu1VKO+t3jqg0AAAAAAAA/FbBZBAAAAAAAgAI2iwAAAAAAAFDAZhEAAAAAAAAKRlXLLl++vCireYizE4k7gWUUzrpkCE4E7cSnMYmBE4s68bRrV41Qvd9Jb1wikNh3J551SYJcvSjaduPo7FPzIFKHO78rq0mW02+6/c5uRe797GPtg89rBM21CRh6Yf78+UVZ9APn+65t7kG50f+d79cK8mNSAOevLq7Mnj27KIv2dzHRJQjplpqkNK6OS5jgbBNjQ7cJy6RyPFwsdWUu7kTbu7a7pC690O141z4YO/rF/v37x6wj+bUyzhnXhpq1WSpjg+ujKzvvvPOKslpcPIptc/PUfc4lqonUxkRn/2hbl9in9lzRL9x49DtxU03yFBdfXZ9qfKP22s7F7/jAchcbnF/UJEx0c7I2sUwtq1evLspie934uj65MYnJtGoTYjlbxzFxn1u4cGFR5tb1mAzOrekxAV4vuHUk2tD5RO06Etvv5nJtEtD4WTcWMWldLbXJ+kaDXxYBAAAAAACggM0iAAAAAAAAFLBZBAAAAAAAgAI2iwAAAAAAAFAwqmp3yZIlRVkUjO7bt6+o89xzzxVlLiFCFIxG8avkxcaO2A4nIHVCVicijsJPJ8Tud4KbmuQ1e/fuLeo4+zvRcLSHE886m51++ulFWUxa4pKYOFu7sYyJM2IiHqn/4nJHFHG7ZD9OfN/P5DXuOyeyDbVzqxdc26Lw3Y2viwVuzsU+uLnqki3UJG6q8VfJtz+ea6J92PXxmWee6Th2c9LFAZeo5qmnnuo4dnHHifvdOMZ6bixcchCX+CAmm3F2djGlF9xaFvvgfLXGFlK5ftb6b00/Y2IQycd4ty7Geet8p98xxSVhqIl3MZmX5NsWkwe5dbh27u7cuXPMz7m10vl17Ldb0/udHM75T7wmcHZ18cIlZdqzZ0/H8aFDh4o6rk/Oz+K8d/7qYogri/7vYny/Y4jrZxxzl7THxZCYzEYq47OLWS72OP+PbXXxwo2lSwQW7e/ikfOxbnFJduK8dNdUzqddH2NsdnZ2cacmSeecOXOKOrVryIIFCzqOFy9eXNSpSVTZDr8sAgAAAAAAQAGbRQAAAAAAAChgswgAAAAAAAAFbBYBAAAAAACgYFTl9tDQUFEWxa5OnO3E/U74GQWjLmnC4cOHizInPo2CUSdsPfPMM4uyc845pyiLgmonwnXi015wgvCYoOKJJ54Ys45UJg6SSiGus6ETLjubxbKVK1cWdc4+++yizAnVY7tq2t4rTlwcE6O4RClOeO2E73HeOP9x4uKaeeME6I6aRB+u7f1OmuDiQ2ybE227MXdJHuK8cfOoNulG7LuLIa4/jpqkGM4Pu2XevHlFWUxQ5pI7ONF+TEQhlfHCxQ8Xn3bv3l02NuDa7gT5zl5xjFy8cutYL7iEAjG21SZN2L59e1EWY3rt/HCJLmI7XCIQl4giJkiQpKVLl3YcL1q0qKgziEQgETfXXCx1ifeifzobupjr6kW/cLZ28cOtlatWreo4drbuNy6ZUEyK5dZnV/bss88WZY8//njHsbvec77u1uKIs7WLBatXry7Koq1dEiKX/K8XnE9FW9esd5JPfBhjeO21tVsP4vrp5pvzneXLlxdlF154Ycexu/7uZzI4t47EeXrgwIGiTm2Csnh9Vns9XkNtwslly5YVZXHM3Bo4Xp/ml0UAAAAAAAAoYLMIAAAAAAAABWwWAQAAAAAAoGDUm4PdQ5yjhsDdj+vue3b3sEcNwZNPPlnU2bJlS1Hm7veOGkKnw3CaBaelXLt2bcexu/e63w/ZdrqqeG+1u1/daYLcPdjRtk6Pde211xZlTuf09NNPdxy7e7nd+Z3uItrR3Ufdb1vXPLjd6YucVsm17ayzzuo4XrhwYVHHaYKcDmnXrl0dx873Xbvcg4XjfHZt7/dDtZ12q0ZD5nRzO3bsKMriODnfj/4q+bkUcdoVN5ZOFxF1SM73XXztFjduscxpVJy9nAYm6hEfe+yxos7mzZuLspoHBjsfcZoRd64a3afTefSC015Hn3ZaOxcnXVnUKrnxcGulqxd92Pmqi3WuXXFNdfOjnz4teY1ftG1tTHT+H9fPGG8lr0901w3RL6JmeCRqYq6zq9Pk9YJrR1yP3brirgHddUnU57o6Lu6767Z4XeLyJtQ8RN3h9Hcu70AvuOujGMtqdfuuXowFO3fuLOo4W7vr9BgLnN+5HB41fu3qOB1jtzg9fI2O22mvXXyN9nLaUOeHrl6Mw86m5513XlHm2h+1k7X66dHgl0UAAAAAAAAoYLMIAAAAAAAABWwWAQAAAAAAoIDNIgAAAAAAABSMmjnECX2jwNaJy2sfNh2F0e6Bli7RgRPAxoetOoG7S2pR88BXlwik34JnJzaN7XXj4ZItuD5FW1999dVFnRtvvLEoc8LxT3/60x3HLoGIa6tL6hKTFTixtkvA0AvuO2KiD9cnJ1R2QvUoqnZCZZc0ySX3icLk2oRSzgfiuZzv9DuZ0KxZs4qymgdt1zwgWirt4ZIhuLjixOqxrU5873zHxcAaXIKQbnExMSaBcP1xba9JqOISj7mECS5OxrY60b5LKuKS8cQxczFmvEL+sXBti9QmeXOJJ6Jtt27dWtSJCYckP59jQhvncy6+umRSMaa4JDj9fni8S7AW44fzaxcnXYKbOJ9d4hqXzMnZ7Morr+w4dmusi69ufKNf1CZJ6wWXFDAm+jh69GhRx8Vqt5bFz9b6nZtL8bpk3bp1RR2HSywSr7NcH9063ws1iaDcfHa4eB191iUTqk38Fq9VLr/88qKO88+nnnqqKIt+7RLQuCRc3VKTIMvNSRfjXb3o58533Lmc7eN4u0RCS5cuLcpcArcYE90+arzXLfyyCAAAAAAAAAVsFgEAAAAAAKCAzSIAAAAAAAAUsFkEAAAAAACAglGzWTiBZRQlO5GyS2DhEs5EwXlMniN5wahLmnDRRRd1HDtxsEvKEAXcUinYdiJ7Z5tecHaMYnsnlHWifSfSj8kIXHICJ7p2AuScc8exS/rhEqzUCGpdnZokROPBtTcmRtm+ffuYdSSfFCAm8HAJn1zSGydUjn3ftm1bUcclznCC5ujXbp7WJJ8ZD66fNd/pbO36FMfSJbZw4x2T/UhlDHHJnVwSF9fWaOva8e6WSy+9tCiLsdOJ6p1NXcKQiJunLnmES3QRY8qyZcuq2uDaGm1YkxClV9w6Fee48zlXVhO/XayOMVjyNovxySUTcn6xYcOGomxoaKjj2MU1N6/6TbyWcP2uTeYU112X4MONkTt/vOZwySjcdYlLPhLH3Plcv3EJPOI1QbfJvCTvsxFnV9euGDOuuuqqoo6LR64sfqeLFy6JTy84n4q+4Wzt1h/nP9Gv3TWhW7dcu84///yOY5fEyl0HucRQ0a/dvqAXH4u4sYznd22o8VWptLNLWOYS/bjrlGhXt8e4+OKLizLnm3Gv4JIguet9l7xoGH5ZBAAAAAAAgAI2iwAAAAAAAFDAZhEAAAAAAAAK2CwCAAAAAABAwagJbnbs2FGUxaQJThDuxLRO3F8jdnWCZJfkY+7cuR3HTrzpksjEZCRSmTTBncsl3ukFZ8cDBw50HB86dKio48ocMTnBI488UtT55Cc/WZS5BDdbt27tOJ4/f35RxyWjcPaPYmMn/HV+0QuubfF7nfjbiZddsqU4b1wCmji2khf3R5s5sX8c25GoSfThklb0gkuoEZP2OLu6ORfnuCQtXry449jFHpdIZsGCBUXZNddc03HsxOSbN28uyjZu3FiUxXnj5ojznW5xPhBjg4vBtQklYvIFl9DAJfFxyamivVxSBeerLnlKbKv7XO38qMWtGREX69zccnEg2sz5iUuU4hLhxL7XJo9wyZ3iXL7kkkuKOm5e9YKzWYwNLsGDs4/z9Zg4xvn+KaecUpS5cYvr4r333lvUcXPE+crChQtHPZb6n3SlJrFV7XWPi9/Rji6eu2RXNUnFHnjggaKOiz3OL2JsdvPNJerqBZeIK7bNxTK3lrmEM3Geu3Fzc8slUor2cOu1myPOP2PSJ9f2fq6Lbt5EW7h2umtOt+bFdX7JkiVFHRdz3TXhrFmzOo7dXLjwwguLMkdM3On6GK+dxoJfFgEAAAAAAKCAzSIAAAAAAAAUsFkEAAAAAACAglE1i9///veLsqiBcPeFuzJ3D3u8B3j58uVFnXgfr+Q1Z5s2beo4dvfDn3322UWZ01jE+6/d/er9vofd6U3iPevuXm6ngXDanthPd7+604I6bUb8TmdXpxOqeQCr0yy6sl5wYxf90+kFnF87jdyWLVs6jh9++OGijhtLp4WKfXdtcLau0Z05+q1ZdMS2Ob2D82unX47nqtXPungUdcPuYbpOf+C0DLHMPWDX6TW6xc2R6Cs1mi3J66CjX7j+OFz8iOPh4oIbR6efjmUu9tVodceD01hGP6zVLDrfj77p4pXTm9RontevX1/UqY1F7iHzkX4/PN6NXbSjWzudzdw1QYwzbn5fcMEFRZn7ztiu7du3F3XcuK1ataooW7NmTcex017126/d2hK1Tw63rriYGOOdW7ecfVwMiX13ukY33vEB85K0du3ajmMX29x1aC84m0WfdfZx16su1sQY5frk1tga3aq75nGauHPPPbcoi+Prrvn7uS46n4646wNXVi31wYAAAAh9SURBVKMhdVpUZxvnT/H87nr8vvvuK8rctUw8l9M/jvdaj18WAQAAAAAAoIDNIgAAAAAAABSwWQQAAAAAAIACNosAAAAAAABQMGqCG/cA6ph4wAk6ax+oHcXTLqmBezh6zYNDnVB6xYoVRZkTl0dxv2t77cNpa3GC2oh7IKvrp2tbtI9LauCSLTghbhRGOx9wOGF0TMzhHhzuEmD0gkuCEcfY2bWm/VKZJOHpp58u6jz66KNFmfPrKHp3omQ3lk7YHcXrTsyecy7KesEljIiCbyfkd/107Y3ncklQXJ/cWMZYUzMnJT9HYkIBJ3J3vt4tLslBbL/zCZcIxCWniOPoYkW3IvraRA7uQfE1DxY+ePDgmHXGQ01SN5cMwcVJF9uin7v1p/ZB9NH+bm12664ri+1wcaffCbLcHIn2cd/pEnw5+8eHXNcmDnLfGcfEjZErc8lH4sO9XVxz86YX3FoWv8PFUhf/3PyNY+muZ5x/uu+sebC6s6uLUTGhipu7zg9dgphaYjI1qbSj82EXd931TOyTGyN3ftf3WM/1uzYhWfQLl7jJxZVu2b17d1EW17zaBHjO9rHM9fmSSy4pyty+I7bDrc1uLXMJJqPvOzvXJDxsh18WAQAAAAAAoIDNIgAAAAAAABSwWQQAAAAAAIACNosAAAAAAABQMGqCGyfyjAJLJ/Z3wuUo2JZKcagT+8dkM5IX68ZzOTGqEzy780cxaEzEI/nEHb3g2htFw07Q7gT5rr3x/DUJDCTvA/H8ToDuxtL5Sk1CBCfE7gUnho+ibScaduJvlyglCrTduDlR9aFDh4qy6OvOXk4I7ZIy1OB8pxdc22KygMOHDxd1apNWRFzscfPezbfoF+5cbt64erHfLunUyy+/XJR1i0uiFOegS9LgbOoSDMRz1SbvcLE6CutdHZeMxNk54mza72RkLhFVTDLh5rxrm/OneC43l10scuMWz++SYbhxc9TE/X6vi25Nij7rvtMlb3BlMQ645B0uJrp68brB+atrg6sX2zXeZBTdUJMAy9nC+Y/z63guF4/cuZzPxrJau7o5EuOPm1v9TtzkYkHsu2uH66dLJuTWt4g7v7N/TNpS46+Sj+uuLNLPdXHHjh1F2VlnndVxXOtzNXPe2cFdC9esSc5XXaKapUuXFmXxmsetp+O91uOXRQAAAAAAAChgswgAAAAAAAAFbBYBAAAAAACggM0iAAAAAAAAFIyaMcIJLLdt29Zx7MSoThzqBP9RKOtEmE4Q65KuRPG0E5o6sbwTmsYkPu77+p00oUag7WzocO2NIl43Rm683bliQgEnVHd+4YS+scwlfum3uLzGF12CGyfsdkkBYgIGZ1eXWMQlsoj2cLZwZU68Hr/Tfa7bxDgj4ZLX1PiPS1rh2hb75Px10aJFRVmNKNyJyd1YuiQ+0S+OHDlS1Ilxphdq/ND5hGu7E/LHfteK410cjud3sa/WN2Of3LxycacXXJ9imWu/S/rhyqLvu7F1fu7GLa6pzj5uLJ2vxHa4ud3v+OH6GW3t1nVX5uJ3XAfdul6bFCjWc/Fj9uzZRZkjxgtn136viy5ZW/SN2sRBzhfjtZzza+eLzv7Rjt0maZLqrkHcehQTv4wHNw/jeNbGLednEdcnh5sjNfHa2b8m1rg+urWyJjGOY8+ePWOey+073DWiG+/oF3v37i3qOHs5n47tcNcabnxcu2ISH9cfN99Hg18WAQAAAAAAoIDNIgAAAAAAABSwWQQAAAAAAICCUTWLW7ZsKcqizsbd4+/u0XX3BdfcZ+7uq3U6m6hRcFoHd9+2a7+7l76mXWvWrBnzcyNRo0tx9x3XPiQ33qft7id3OhVnn3hvvbuf3z242mnTYj03Rq5dveD6XvOQX3cPeY121X2f809H1Ko4nUrNGDlcHfeg7V5wvvH88893HD/zzDNFHac1qHlYbzy35Pvpxjfe5+/0ZC6O1ei7auNRt9ToW2p0jVKdrsqNRe2DseNcc/G81qfj/KjVv/aCO19sb63exFGjVXe6yZo46cbI6YGcr0SNovPp2j7Wsm/fvqIstrf2gfXO15csWdJx7LSBzhZuTGI7nC86H665DnLz262xvejo3JyOMcrZulZn6Moibv1x4xZt5sbDjaWzY+x3rWaxF1zbYjtqHxZfEz9dvHDnr51LERd7XB+jzzof7qfu2e1h4lief/75RR23BtbkAnHxr/b6I8Y1dy53/VETc10scmWjwS+LAAAAAAAAUMBmEQAAAAAAAArYLAIAAAAAAEABm0UAAAAAAAAoGDXBzXPPPVeUxQd0xgdZj1TmBLZROO7ExzUJRKRSKOuE0q4NTjBa0y6XSKMXnKg6CpediNuJv2vE9+5zDifuj0JsZ+uaB8y7c/X7QcO11DzA3Nm12yQxTpTuPlfr/zXEMXHn7vdDtV1igJgk69lnny3q7N+/vyhzSaXiudx4xCQWkhd3x+90DzuuFZi7BwtHaudgDdEO7vzu+1xZjQ+4PruECc7P4xi5xAGuXS5+xLLaB1n3gvOdaA/nh65tNckPXDIEZ4uaRFourtUmBYrf6daGAwcOFGW94OZRjAO1SZqcL8Y+uPFwSUXcmMR54xLG1K6L0f4xuZDkk4Oce+65RVktNeuza3+3SVEctbEg4tpek2BFKv3fjVtNcrV+4643apIhuc+6JFZuPrj4EO1fe+3ibBbnl/u+ftp6165dRVnsj1vTaxO4xf2Qs42L+67fMc64Ndad38XvGNdq1uGx4JdFAAAAAAAAKGCzCAAAAAAAAAVsFgEAAAAAAKCAzSIAAAAAAAAUpJpkHAAAAAAAAPDTBb8sAgAAAAAAQAGbRQAAAAAAAChgswgAAAAAAAAFbBYBAAAAAACggM0iAAAAAAAAFLBZBAAAAAAAgIL/DwQY/7/XOymfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_test_with_title(worst_predicted(0), \"Worst predicted bad CO-tips\")\n",
    "save_name = f'worst_predicted_bads.png'\n",
    "plt.savefig(save_dir+save_name, bbox_inches='tight', dpi=200)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Worst predicted good CO-tips\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAABzCAYAAAAxOpeKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2de9Bd1Xnen4VtsEBgXT8QuhhL6AYCIiMD03E9nozxYMg0tgfaGqbEU48zOKV204mnrifpyXHSjFOmk0vbpCl1mjAOJgN1WyhuARMLULARQgiQuAgJJKE7F0nohm3M7h/nKPnOu57vO+s7Z2tLob/fjMfs91v7nLXfvS576eznWamqKgEAAAAAAACM5pQTXQEAAAAAAAA4+WCxCAAAAAAAABksFgEAAAAAACCDxSIAAAAAAABksFgEAAAAAACADBaLAAAAAAAAkPHeYT8gtdMUSddXreqPaqjPoHX4U0m/IGlv1aqWjVEmSfoDSVdLOiLp81WrWtv92y9J+vVu0d+uWtWfH/9aTxxy3RzkujnIdXOQ62Ygz81BrpuDXDcHuW4Oct2fOn5ZnCLpV9wfUjsNvRgt5M8kXdWnzKckLez+75cl/bEkpXaaJqkl6XJJl0lqpXaaetxqOhzkujnIdXOQ6+Yg181AnpuDXDcHuW4Oct0c5LoPdSThm5IWpHZaJ+kBSfdK+i1J+yQtSe30SUn/+9hKObXTr0maXLWq30zttEDSf5I0U51V8herVvX8RCtQtaqHUzud16fYL0q6rWpVlaQfpXaaktpplqSPS3qgalVvdOv3gDo37DsTrUcDkOvmINfNQa6bg1w3A3luDnLdHOS6Och1c5DrPtSxWPyapGVVq/o5SUrt9HFJH+7GXu5z8f9F0k1Vq3oxtdPlkv5I0s+ndrpB0ldN+U1Vq7p2wHrOlvTKqOPt3dhY8ZMRct0c5Lo5yHVzkOtmIM/NQa6bg1w3B7luDnLdh+P18+rqqlW9PF6B1E6TJf09SXemdjoWPk2Sqlb1F5L+4jjV7d0GuW4Oct0c5Lo5yHUzkOfmINfNQa6bg1w3B7kexfFaLB4e9d9vq1cb+f7u/58iaf+xlfxojtOKfIekuaOO53RjO9T5CXd0fOWA33EiINfNQa6bg1w3B7luBvLcHOS6Och1c5Dr5iDXo6hjsXhQ0pnj/H2PpJHUTtMlHVLH7ef/Vq3qzdROL6d2uq5qVXemjsvPxVWreuo4rcjvlnRzaqc71BGBHqha1a7UTvdJ+p30t2LQT0r61zV/d12Q6+Yg181BrpuDXDcDeW4Oct0c5Lo5yHVzkOs+DO2GWrWq1yX9dWqn9amdbjF//6mkb0harY5wdLTw8wZJX0jt9JSkDeqINydMaqfvSPqhpMWpnbandvpCN35TaqebusW+J+klSZsk3aqu81HVEYT+lqTHu//7Rjd20kGum4NcNwe5bg5y3QzkuTnIdXOQ6+Yg181BrgvqV1VVnZ8HAAAAAAAA7wLq2GcRAAAAAAAA3mWwWAQAAAAAAIAMFosAAAAAAACQcby2zhiY1E5bJK2oWtVrEzzvNEm3SbpU0uuS/lHVqraYcl+R9EVJSdKtVav6/W78Ekn/WdJkSVsk3dB1OjpV0p9IWiHpHUlfqVrVykGu7WSDXDcHuW4G8twcQ+T6Y5J+X9LFkv5x1aruGqPcpZL+TNIkdYT9X6laVZXaaZqkv5R0njq5/odVq9rXdaL7A0lXSzoi6fNVq1o78Ss7+SDXzUGum4NcNwN5bo53a64b+WUxtVMTi9IvSNpXtarzJf2epN819VimzoPeZZIukfQLqZ3O7/75v0r6WtWqLpL0P/S3+6N8UZK68Ssl/fvUTiftL7LkujnIdTOQ5+ZoKNfbJH1e0u19yv2xOvlb2P3fVd341yQ9WLWqhZIe7B5L0qdGlf3l7vknLeS6Och1c5DrZiDPzUGua/hlMbXTIXUsXD8pabc6K+JXUzutlLRO0kclfSe1023q/Mv7vO6p/6JqVX+dOvuWfEfSbHVsY9OAVflFSb/Z/e+7JP3H1E6pavXYvS6V9FjVqo506/6QpM9K+neSFkl6uFvuAUn3SfoNSRdI+itJqlrV3tRO+9X5lWD1gPUcGHLdHOS6Gchzc5wsuT72i21qp3fGqessSWdVrepH3ePbJH1a0v9R5159vFv0z9XZfPhfdeO3de/Zj1I7TUntNKtqVbsGqecwkOvmINfNQa6bgTw3B7kuo47V8hmS1lSt6ldTO/0bSS1JN3f/dmrVqlZIUmqn2yX9XtWqVqV2mqfOw9TSbvlVVav6Rmqna9T5F351z3lEfqPMX6ta1fdDbLakVySpalVvp3Y6IGm6pNE/Ba+X9G+7N/eoOj/Lrun+7dj+KP9T0nWS5nbjT0n6B6mzB8pcdV5Tm6sT8LAnct0k5LoZyHNznCy5LmG2pO2jjrd3Y5J09qiJbreks0ed84o5p/EHEJHrJiHXzUGum4E8Nwe5LqCOxeI76rwnK0nflvTdUX/7y1H//QlJF6T23yy6z0rtNFnSx9T513lVrere1E77jhWoWtXfr6F+f0PVqp5L7fS7ku6XdFidfzX4WffP/1TSH6Z2+g1Jd0v6STf+p+o0iDWStkp6dNQ5TUOum4NcNwN5bo6/M7kupepoNU7GzYLJdXOQ6+Yg181AnpuDXBdwPN7DHV3Bw6P++xRJV1St6q3RhUclPmOCq/Id6vyL/fbUeb/4A+oYVfRWrlV9S9K3up//O+qu0qtW9bw6P0MrtdMiSdd0429L+tVRdXpU0sYxK90s5Lo5yHUzkOfmOFG5LmGHpDmjjud0Y5K0J3Vfo0md13L2jjpn7hjnnGjIdXOQ6+Yg181AnpuDXBvqWCyeIulaSXdIul7SqjHK3S/pn0u6RZJSO/1c1arWqaPzuV7Sb6d2+pSkqcdOmOCq/G5Jv6TOO8PXSvqrqldvpO73jlQd7dA8df414IoQP0XSr6vzbrJSO50uKVWt6nBqpyslvV21qmcnUK86IdfNQa6bgTw3x8mS6750J7w3UztdIekxSTdK+g/dPx+7V9/s/v//GhW/ObXTHZIul3SgOgEamC7kujnIdXOQ62Ygz81Brguow5XvsKTLUjutl/Tzkr4xRrkvS1qR2unp1E7PSrqpG29L+lhqpw3qPHxtG7Ae35I0PbXTJkn/Ul0noNRO56Z2+t6ocv+9+/33SPpnVava341/LrXTRknPS9op6b914yOS1qZ2ek4dseg/GbB+dUCum4NcNwN5bo6TItepnT6S2mm7OtrOP+l+3rG/rRtV9FfUcZndJGmzOiJ+qTMZXpna6UV1Xg36Zjf+PUkvdcvf2j3/REGum4NcNwe5bgby3BzkuqR+Vf6P5xP7gHY6VLWqyUN9CBRBrpuDXDcDeW4Oct0c5Lo5yHVzkOtmIM/NQa7LOGn3+wIAAAAAAIATx9C/LAIAAAAAAMC7D35ZBAAAAAAAgAwWiwAAAAAAAJAx7tYZt99+e/aO6vve976e45kzZ2bnTZs2LYudeWa+1cgpp/SuVX/6059mZd56660s9rOf5Xta//jHPx73eKzPct8ZP9991pEjR7LYjTfeOPaGK31Yt25dluuYn3feeSc7z+XiJz/5SRaL9T169GjReS4W6/H222/3LTMW731v/91bYh4k6frrrx841/fcc0/fd69Tyj/e1dXFYn1d/d15p556ahabNGlS3zKxT0rS+9///r7f6e7tli1bsthHP/rRgXPdarX6jiGzZs3Kzps3b14WmzFjRhY7/fTTe45LxgaprN+7MoOOK66PuPMGHUPuv//+LM/xelxuSvtpLOc+y+XLtf3YNl1bdW26pJ27z3rPe96TxZYuXVrrWB1xeXVtwPVBVy7i8upiETd+TJ6cezu4+TrmtrQOU6dOHTjXd9xxR9/xw32na4sl85u7b+6zSsYPh2uLg+L6w5e+9KWBc33LLbdkuY7t5bTTTsvOc/kv6b8u125edHNxjDk5lYuVPM+U9t2bbrpp4Fx//etfzyoX+9cZZ5yRnefy6oj3xJ3nrtPlrKTNlrbr+PluPnXPpl/96lcHyvW8efOyC7rhhht6jj/72c9m5y1dujSLuXEyjgOu7ocPH85iJc8Mbo1x8ODBLHbgwIG+5UrrcPPNN4+ZZ35ZBAAAAAAAgAwWiwAAAAAAAJDBYhEAAAAAAAAyWCwCAAAAAABAxrjuIpdeemkWi4JnJ5x1ItkSgwcn6Cw1lIixUgG6E/nG73Tf5+o1DM6YocTgptQgocT8wJ1XGhvk+9xnlZ53vHGC7RIzG6ncNCRSIuR39XJtJxrjSHlfdX13+vTpfes5Ea666qq+9YgmNWPFHLGfD2pi5cqVGuMMer8HPc/hrqfEcMNRYuhRYhQhDd6fXd1d248xV6buMaXElMvlojQWc1uaa0ecr12/KrkeKZ+fSw2/pk6dWvT5DlffkvtZ0lYcLteD9q1hzGziNTbRrp35Ufze0n253TNC/CxXxs2BjpLPKu0jcT5yc0jd+5E7s7bYd5yZUOlzScTl1d3vkjZVakJU8hzqyrh1wKDMnj07i33gAx/oOS7Ns3seiMYxg5rZuHLus0pN5OK9LTXpGo+T48kcAAAAAAAATipYLAIAAAAAAEAGi0UAAAAAAADIGPdl53PPPTeLxXeT3XvhbmNK995u3Dhy3759fctI0qFDh7JY1BAM+m63lL+b7N7tLdHtTYRBN1ku1Y3E+rocus093X2L97zknWnJ3xNXrt/3DYvTpcScleq9BtWzuOsu2YQ1vm8vlW18LpXpLtz7+8OwaNGiLFaixXV9zuUntlnXht15JX26VDdX0t9KdQuDUqKPGlTDKOX6CTcuu2t0xM2mnS7NbQrvrrFkk+q6tV0l+jGX65JN4SXpzTff7DkubdNuTorjgNNKlW5CHscsN4a56xkGd+/q3Ng+1tfl2s2VJX23VDfp8liygX3d7dptAj+oZtHVd1CPCvedsV27ecvp9kueN9wcW3e7Lrnnbmxz2kA3/5T0kVJ9aCzn2r6rg7tvcW1wvP1ApkyZksVi23TaQDcOuGuM57o1jPss951xznD3p9QPJrZX1x8nOo7yyyIAAAAAAABksFgEAAAAAACADBaLAAAAAAAAkMFiEQAAAAAAADLGdXxxAuEounQiZRd7/fXXs9jOnTt7jrdv356V2b17dxZzgvMoPj3rrLOyMk647IxA4jU6UWndpislOEF7iUGCJO3atavneNu2bVmZeD8kaf/+/VksCmPdpstuc/dp06ZlsUjpxvfDULo5dsQJr13+S0TczgDDCZpjOy4xnpC8aUj8fJdr12+GwbWDiMtrqanHa6+91nPsxos9e/YUfX7Emay4MaRk43BnCjCM4UzEmSGUmL84ob2LxRy6scKN8Y5opDEyMpKVceZqbqx2sZOBEoMPyecsjs0vv/xyVubVV1/NYm4siu11wYIFWZn58+dnsbPPPjuLxXbucl9qIleK+44SEy43ppQ8q7j74cYPZ+IX+7wbK9xc6Z6zYt8d1NxpIrh5Nt5P9yxU+lwSc+vasMu1u5dxDHHzzDnnnJPF3DNIbGOlxi/D4OaM2A7cvF76fBSfE0rKSP6+lZhHOkoMeupuwxHXduL6wfVld54zdduxY0fPsXuu3rt3bxZzY1HEGU65Z5LJkydnsXiPXFua6BqGXxYBAAAAAAAgg8UiAAAAAAAAZLBYBAAAAAAAgAwWiwAAAAAAAJAxrhrdiWmj2NUJNZ0pyiuvvJLFnnnmmZ7jp59+OivjxP1OfBrF5DNnzszKzJ49O4s5IX8UoTtBb53mFGMRBaglZiqSF45v3Lix53jNmjVZmRdffDGLOVOGaIIya9asrIwTl8+bNy+LxXOdKUDdpgklwl4n/na53rdvXxaLQn4ncHYmIo7Yjp25kxOvO2OIKHJ2ea071yW4McTlbPPmzVksjiHxWPKic3cvo1Dcteu5c+dmMWfGEj+r1BSgTuJ3OpMdN5Y685qY1/Xr12dlnLmQI5pRzJkzJyuzZMmSLFZi7uBMAZrIfayHG2Nc33W5fuKJJ3qO161bV3Sey0+c39x45fqCG//i/FlqmjEM7jtibl27dgZWrn3GOc+1azd+uDErzl1uXHBt3Y0zsY84Y4u6ceY7JXOlaz8u/1u3bu05fumll7Iy7tnR9Zs4brlnu4ULF2YxZ+YUn1VcG67b0NCNSbGtu7ZfYkLkYu5+lBqvRDMeZxJUarwSc+ueL+s0NCwx0HHP1S43bvwomRfdGsZ9Z8yza9Mf/OAHs5gbZ2JfruNZj18WAQAAAAAAIIPFIgAAAAAAAGSwWAQAAAAAAIAMFosAAAAAAACQMa7C0YlPq6rqOXZGHU4c+uyzz2axhx56qOf4ySefzMo4cXMUgkq5gPe1117LykTjEUk6//zzs1jEiXdjHobFiapLTBOcIPyFF17IYqtWreo5Xrt2bVbm4MGDWWzSpElZLAra33zzzayMizlRdxRBO4OKOgXPYxHz74w/XK537NiRxTZt2tRzHIX9Y33WaaedlsWioYoTRjuhsjO4ie3YtesSQfhEcG023k+X6z179mQxZ16zcuXKnuPHH388K+PaostZFO67McQZhLhxMgrRnXFTncYr7rNiG3D31plhudw/99xzPcduPHf32o3VEXf/nWmJ+6wTMX64usX25MZzZzLhTMWiwU0cTyR/L6dMmZLF4vzsvs/V1fWPmP8Sw6FhKekjzmzOGXW4sTo+c7h50Rm4ODOY2I537dqVlXHPDS6P0eDGmYXUPVa7XJe0a9cfnKFhNLRx850z63B5jGOUe7ZzeXVmQvGeuDEkpZTFhsH1E/esFXHtx7X/+Cy3evXqrMyGDRuy2BtvvJHFYjtbvnx5VuYjH/lIFivpu67t1ImrQ2x3bt5y86J7/vjBD37Qc+zGDzfuu3sd+7hbR7nYsmXLstiCBQt6jt14NdGxml8WAQAAAAAAIIPFIgAAAAAAAGSwWAQAAAAAAICMcTWLTlcVN/Yu3Wg4al5czGlXFi9enMVmzJiRxaK+yG2g6TR5jqjNcNqAujd6dpqEiNMLuGvavn17FnMagsjFF1+cxdw7/vEdbPeeu9OyOu1B3DR63rx5WRm3CewwuPfk43vrTvPiNGxOTxQ1R1u2bMnKuPYzc+bMLBa1Eq6PlG6qHd/Vj315rFhJ2xwLNz7Eerg27NqK08lFHYH7PrcRs2tTcbxz+kTXLty7/1H3UqLdGwansYh9y7V7N8a79hpjThN7wQUXZDGnQYq6CzdeuTnEabGj7tbplIZpvw433sU5yWmLXa7duBzvZbyPktcIRZ2blG8IvXnz5qyM0zG6nMXxyWm7JrrRcz/cOBnHKKfZcuOkG1O2bdvW9/ucJihu5C7l2mg3X7jnEqdfivfc6ctdbBjcnBHnH6fPdu3aXWccM0rHC1evO++8s+f40Ucfzcq4cWzJkiVZLD7juPZUN06vGecR15dc+3T1jfpcN8a652hXr8cee6zn2N1v9+zi+kicB50WtE4/EKfjHhkZ6XueG5fXrVuXxaJG0c0N0b9A8rmJz9VOn+j8F1y+4n1083WJRnY0/LIIAAAAAAAAGSwWAQAAAAAAIIPFIgAAAAAAAGSwWAQAAAAAAICMcdXoTqx51lln9Rw7IxNntuA2xI0i9KVLl2ZlPv3pT2exuFG5JD3yyCM9xw8++GBWxgncnUg8Gls48a4TjA6DE3FHnGjfGXq4/Edh+sKFC7My1157bRa74oorstiaNWt6jr/73e9mZaKxguRz5gTzkSY2io8xZwbi+sPWrVuzWBRHO1G6a+uuXcd6OIFzNGmQvNlFNA9yxhl1mya4nMUN6l0bdpssO8F3bD/ONMGNIXHTWik3SXjggQeyMs60wtUritVde6pzDIkbXktlhkbOsMcZDsU27MaPz33uc1lsxYoVWSwaJtx1111ZGdevXCwaYjiDjLrNhZwhTMSZUzjTGzd/xnZx0UUXZWVuvPHGLObGlHvuuafn+Nvf/nZWxuXVjSlxI3RnFOEMMobB5SfG3KbXrv6uj8Rz58yZk5W5+uqrs5jbmHz9+vU9x/fee29WxuXazf1xjHS5dmP8MDijDNdmI85gxV1TfHZ0890nPvGJLObmqTivuM3Q3dzjjLPOO++8nmNnilL3817JM0ip0Ytr/xs3buw5doZV1113XRZz9+TLX/5yz/HKlSuzMs58LuZVyu+lazslba4UN0/FPu4M6tzzh2s78ZncGSh95jOfyWJu/Fi9enXP8d13352VccY7zrwo5t6Z7MT+2A9+WQQAAAAAAIAMFosAAAAAAACQwWIRAAAAAAAAMlgsAgAAAAAAQMa4BjfOUCUKUp2Q2Z1XIuJ25gROcO6Es9FQxYmUnTmFExpHwaszK3BmEcPgjEvi9zozimjGI3lhd6xvNDuRpMWLF2cxZxgSv3Pq1KlZGWdw4+ofY87wxt2jYXCfF0XVpbnevXt3Fot9xOX1yiuvzGIf/vCHs9jmzZt7jh966KGsjBNjO9FzrKsTPdfdrp34PrZrd89d/p3ZRTQ/cmYIzsxm2bJlWSz2m8cffzwr40yy3DXGmBsT6zRNcKZWs2bN6jl2JhlO3O+IYng3Lrvxe/78+VkstkNnIOaMQNz9j+3EtRtnLjQMbmyLZiMuF+5+O0OYOHe5Nu3GXGd4MmPGjJ5jZ3Th2oAzmYgx129L21Mp7t45w7ASnJlGvCfnnHNOVsY9b7jxI9Y1GsFJ3qDCEcc1Z/JWt/Gba4vxO1wO3fORm2NjuUmTJmVl3PzjjDhiW4/tXPJt3Y018fNdP63b4MblrOS505neuFg8131WnB8kb6YVjbPWrVuXlSltn/H+uv7t6joo7vk19nH3/ORM3tzzX+wzLqdu/HBmZPH5o/S52q0d4lxZxxzIL4sAAAAAAACQwWIRAAAAAAAAMlgsAgAAAAAAQAaLRQAAAAAAAMgYV0nqBOxROOkMBewXFQh6nYHBhg0bspgz73jppZf61iulNFC9BhXUTwRXjxhz4u8zzzwzizlhbDTccLlwphLOmCiKbJ2pgauXE6pHMwcnVJ82bVoWGwZX3xJjBicSdkYQUbg/d+7crIwzs7nmmmuy2BNPPNFz/MILL2Rl9u7dm8WcaDvGXH9zbWcYSnLmTBPc/XD98PTTT+973saNG7PYnj17stjzzz8/bj0lbwIR6+BiTuxfpxmIu579+/f3HDvxvTMCc0Yp0RzMGVjEMVjyc0gcZ1wbcfkqMYhxZeoev52BWJxvXD2cuYYzr4n92Y3Bbl505imxnGvTzgjEtYum27Tk21mJ+Yvrk+46Y8zNi1u2bMlirs1u27at59jlOpr6jVWvWP8mTFcGvXcu126uj+ORGy/cc9APf/jDLBbNg1xencmUG9ti/d311E3MhSSdccYZPceHDh3KyjijNDfGxjn1xRdfzMrceuutWcy1gWj05kyg3HjhiP3G9ZGjR48WfVYJs2fPzmJxPnDja6mpYrwfzmzm2WefzWKu3KZNm3qOnXGeq6ub3+J9dNczUdMbflkEAAAAAACADBaLAAAAAAAAkMFiEQAAAAAAADLG1Sy6d1qjXsO9L+00BE7DFtm5c2fRZ7l39aNmx73v6z7L6eHi+9funWD37vAwuHfR43vG7n1+p/Fzm63HjbDdJt5OD+e0UCW5djoMtzlprKtrJ6Xvj5fi3suPbcrpQ9157p5E7UE8lvy7+k4b47SHEaezKXlHvWTj7eNB1He5vuT6uGvrsS06/eb69euzmLuXUZfndM9OG+N0EVEf6M6rc1Ntpz+N99J9n9P1LFiwIIvFsdnl+Zlnnslirk3H8cJ9lutXTt8X9TNu/KhbR+f0JiU6G6frcfmP+iKnRXTX5MasHTt29By7jaXdea6usZzTGjud1fGmdKxw80/s8y4/Tu/l5s/YLlybcPPihz70oSw2f/78nmOnE6tbW+e0uHEecffcXaeb8+IYsnbt2qyMy7Ubh+OY4cZXN164nMXnO3eNdePm55hH18fdXOme0+Oc5J7jHnnkkSzmvjN6MCxfvjwrc/7552cxl+tYf1f3OvPv6hCv0c01bh5xa4XonxKfsyXp6aefzmJuXoz3yPVHdz3OiyCOF65/OP+F8eCXRQAAAAAAAMhgsQgAAAAAAAAZLBYBAAAAAAAgg8UiAAAAAAAAZIxrcOPE6lFsfPDgwayME04605Vo3hHFopI3XXGC1CicdWYOc+bMyWKLFi3KYlEY7a5nohta9sPlOops3TW5jZ4XL16cxaIg3AnJ4wbwkheqx1w7QbITPLuN6KOZhtvMt4mNnmPMlSnZ/FTK8+EMMdxG8c7MZteuXX3LVFWVxUpMgZxY3hktDYMzJygxknFCbmdQEQ0p4sbYkt/A3JlixPvmxhln/rJs2bIsNm/evJ5jZ2xRJ66fxhw60b4Tvl988cVZLI4fTz31VFbGGQm5PMex07W5mD9JWrJkSRYbGRnpOXZt2vW/YXDmL3Hccnl1MTcmxly7cfnJJ5/MYs6oJpocufvhDFYuvPDCLBaNFNzYV7dBlptn43zs5mdnCHPRRRdlsWgq4ubFaBIkeSOc2PZcrt344frbBRdc0HNcuun58cbNgW6sds9asZ87k49Sk8OpU6f2HLv8uGc71y6iQZwbQ+p+Bil5bnbtp9TYKubjsssuy8q4Z3I3Fsd5wxmqxHFY8s8gca53/bvELKwUl694ja6Mux43/8Rnma1bt2Zl3Pjt5uL4/OHaoRurL7nkkiwWx5k6jCP5ZREAAAAAAAAyWCwCAAAAAABABotFAAAAAAAAyGCxCAAAAAAAABnjGtw4sXEUdDqR5KRJk7LY3Llzs1gUFkfRsiTt3r07izlDlSiyjiY1kheHOiF2NKNwIty6jUBKxKZOiOtME5zhRjTAcAYSTpzrjHei8NrdW2ey4wxKpk2b1nPsxOx1mwm5PMbvdYJg16Zc+4nmNa4fbdq0KYtFYwtJ2r9/f8+xM7hxuL4URe9OGO/61jA4g6o4hrj74UwTXJuNuOuO3yf5NhXvuTNDcKYJrl6u/pG33nqrb5lSXNs899xze46dGZYzUXCUGKU4cxBnLhPz7Oo1fwAwWLgAAAXsSURBVP78LOYMGeL44Qw46jRMcN8p5dcwY8aMrIzrb65u0fzCGS05g6zDhw/3revs2bOzMm6sduNavOfOzMaZMgyDG4/iPXZ5dTF3nXGud3OUM7hxfTc+97gcLly4MIu5exLr5dp13fOiu5/xGS0eS95gzY0P8TnBGRO5Nuw+P+bHPQc5szxXr5hb92xX97zorj22Wfcs5NqBa+tx/nFjrDMhdM+hcfxxdXf1cmNBbLOuPbnxblDc+Brvpcufy1c0nXKf5dqhM91zbSw+o7v740y6li5dmsXc/BOZqGkTvywCAAAAAABABotFAAAAAAAAyGCxCAAAAAAAABksFgEAAAAAACBjXCXpqlWrslgUazrBsDMicGYR0RhiZGQkK+MMEpzYOIpunUGJE586cWvkeItwJS8GjiJY953OSMMJu6Ng15n97Nu3L4s5IX/MWam43BkfxWtyIvu6zYSmT5/et4xrF878IJpRSHnOnMHKhg0bspgTJcd24dq+MxhyYuwFCxb0HEfDG8mL6ofh+9//fhaLhhGur7r26a4zXlM8lrxJkzOHiGOIa9fO3MQJ/uN9cv17ogLz8XDmHdEkxtXd3W/XxqLxgWs7zlgrGjQ5orBf8m3CjXXRWMHd17pNVy688MIsFtudq7/LqxuLli9f3nNcamLmxu84jrn75sZq1zbj2Hy82/RYlJiNuLnGXWccc50BzYEDB7KYa2exL7lcu7ZeMuY645G6DW6cucyRI0cG+k73rBKfFUsMyySfnxLTFdcGSkzdXK5jHobFPdOUmBy6+rtrj/3Q3Q8Xc8+6JZ/lKLlv7vnJmeUMihsrnGFSxLWdEiM7t/bZtWtXFnP3P36nWzM5Izs3rsU8u3G5pL31fMaESgMAAAAAAMD/F7BYBAAAAAAAgAwWiwAAAAAAAJAx7svH9913XxbbvXt3z7HTwTjtRMkm7U6/4XRKg+oF3Tu67p3mEo2Le7d7GJxeIL7j7947drkoeX/fbTpa+g5znRqg+O620wbUrRcoqb/Lq2ufbuPUWN/169dnZd54440stnPnziwWtU+zZs3KypT0LSlvF07LN9H32Pvx8MMPZ7E4hri8Oi2Du2+LFi3qOXa6ApcLp4sYVG9VkjNXpk5txuWXX57F4tjpNH+uXq5c1Eq4Mm6sLtH0Op1yKSWaXqerH4YVK1ZkMacvKcHlMero3Fjk9M1unIy5Ltko253nznWfNcy9dDjdU4w5fVHctF3yuY5jYp19shTXZuO8WKq7HgY3FkSNpdN7uXbnPiveJ/dZJfo7qewZ0J1Xond11D2GuOfH2K5dGdc+XSx+lsuXy7+LxZyVaNrHisVzjx49mpWp06PCrUVivVw9S58F4vOZG6/c3FCiDXX61BIdqMON1WgWAQAAAAAAYGhYLAIAAAAAAEAGi0UAAAAAAADIYLEIAAAAAAAAGeMqI1999dUsFoWZTqi5d+/eLOZEuFGY6TasdUJ+J1SPwlwnknWxEiGrE5KXbOw5EUpE3KUba7pyMbcu16WmPfHaXR2coNYJ4aPA2d2j119/vahepZQIx52QuGRDX4fbnNn1LWcqEY0anFg6br4uSVOnTs1i8T41YXDjNrSO1+T6sztvy5YtWSye6657ZGSkbx0crt+XxmL7cfe2TpzRWBwHXP8rNTeJ5UoNglw/Kvms0jzH8aNO862xcHNSxNW11BAmjq8uh24j5pK5zLUBd567J7H+pdc4DCUmLqX1d8RyblxwpiuOEuOMEjMbF3NGUW6z72EoyZnLhZtPBzXdGGbz+EGJn+We7ZwRyzC47yhp6y5n7p7Etuc+27XPQecpV68SAx1XrzoNbly/ibi2NOjzvTPbcjFH/M7SObZknCk1IBoPflkEAAAAAACADBaLAAAAAAAAkMFiEQAAAAAAADJYLAIAAAAAAEBGqtuoBQAAAAAAAP7uwy+LAAAAAAAAkMFiEQAAAAAAADJYLAIAAAAAAEAGi0UAAAAAAADIYLEIAAAAAAAAGSwWAQAAAAAAIOP/AfsSpwPrSmm4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_test_with_title(worst_predicted(1),  \"Worst predicted good CO-tips\")\n",
    "save_name = f'worst_predicted_goods.png'\n",
    "plt.savefig(save_dir+save_name, bbox_inches='tight', dpi=200)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "most_uncertain =  [33 32 15  1 31 19  6 21 23 24]\n",
      "Most uncertain predictions\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAABzCAYAAAAxOpeKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2de9Bd1XnenwUCgblJSAIkBEISd1BgzL0Jri+ZlMZu7Ni0jS8pzaTTcUNjponTsXtxEmfakriFkmQS13UaXMepcTBOE5qhxsE0dlNMbBCOEDdxEYibEOImZIOB1T/OlnTOu57v+97vnKOjT8rvN6Ph7PWtvc/a737Xu/fm7OfZpdYqAAAAAAAAgH7229MDAAAAAAAAgLkHN4sAAAAAAADQwM0iAAAAAAAANHCzCAAAAAAAAA3cLAIAAAAAAEADN4sAAAAAAADQMPrNYikLVMrPjWEso4zhEpVyn0rZoFI+NkWf+Srluq7Pt1TKCX1/+3jXfp9K+TuTGfQQEOvJQawnB7GeHMR6MhDnyUGsJwexnhzEenIQ65mptY72TzqhSuum+Nu8kbc/8/fvX6UHq7SqSgdW6a4qnW76/VyVPt19/qkqXdd9Pr1bZ36VVnbb2n+3j5tYE2tiTayJ9b4Za+JMrIk1sSbWxHofifU4HkO9UtJqlbJWpXxKpbxVpXxDpfyJpPUq5QSVsm5n71I+qlJ+pfu8WqXcpFK+061z6hDff76kDar1IdX6qqQvSnq36fduSZ/rPl8v6R0qpXTtX1Str6jWhyVt6LY5FyHWk4NYTw5iPTmI9WQgzpODWE8OYj05iPXkINYzMG8M2/iYpDNV69mSpFLeKunNXdvD6v+ZtOUzkj6sWh9QKRdI+h1Jb1cpH5T0S6b/BtV6aWg7VtJjfcubJF1g1t3Vr9bXVMoLkhZ17beF9Y+dZsx7EmI9OYj15CDWk4NYTwbiPDmI9eQg1pODWE8OYj0D47hZdNze3d1OTSmHSvpbkv5IpexonS9JqvULkr6wm8a2r0GsJwexnhzEenIQ68lAnCcHsZ4cxHpyEOvJQaz72F03iy/3fX5Ng0Y6B3X/3U/S8zvv5PuZ3R3545KO61te3rVFdvTbpFLmSTpC0rOzWH+uQqwnB7GeHMR6chDryUCcJwexnhzEenIQ68lBrPsZgzBzUZU29i2/tUo39i0fUKUtXb/5VbqtSr/S/e0vq/T3u8+lSmcN8f3zqvRQJ+rcIQw9w/S7PAhDv9R9PiMIQx+awyJcYk2siTWxJtZzPdbEmVgTa2JNrIn1PhLr0Q1uan1W0v9VKetUyqfM338g6ZOSbpd0s6R7+/76QUk/q1LuknS3vKBzpu9/TdI/l/S/Jd0j6Uuq9W5JUimfVCk/0fX8PUmLVMoGSb+g3jPK6vp+SdJ6STdJuly1vj7rcUwCYj05iPXkINaTg1hPBuI8OYj15CDWk4NYTw5iPSOl1jrO7QEAAAAAAMA+wDhenQEAAAAAAAD7GNwsAgAAAAAAQAM3iwAAAAAAANAw924WS3lEpSweYr35KuU6lbJBpXxL071Es5T9VcqdKuXGvrZrVcrDKmVt92/HyzkXqpSvqJTvqpTbVcqZsx7bXGVPxXrX335TpWzrW16hUv68i/WtKmX5rMc2V5l7sf6wSvnrLte/qVJOn/XY5iK7O8697e+I27f72s9SKf+v+9ufqpTDu/ZFKuXrKmWbSvntYXZpzjL3Yn2CSvleXw3/9BB7NTfZc+fFld16G7rtHNi175v1QyLWk2T315AFKuV6lXKvSrlHpVzU97ef79rvVim/0bWd31c/7lIpPznMbs05dmecSzmuO8et72J5Rd/fruuL5yMqZW3XznmxXW/UWP+aetfOa1XKV1XKsq79iO48eVe3zs8Ms1uTuVnsvQ9kd/Ozkp5TrSdKulrSr0/T9wr1HIciv6Raz+7+re3a/pWktar1hyT9I0nXjHPQY2dviXUp50paGFr/o6T/3sX6k5L+w1hGu7vYu2P9h6p1jXrvB/oNSVeNZ7i7gbkX57d1NeLcvrbPSvqYal0j6Sva9X6l70v6t5I+uhvGPH727lhL0oN9NfzDYx/5OJl7sXb149clXd2t/1y3PWlvqh8SsZ4kcyvW10i6SbWeKuks7Yh5KW9Tz7XyLNV6hnrXHpK0TtK5XawvkfRfJrQ/s2fuxPk1Sb+oWk+XdKGky3f+D41a/+HOeix9WdIN3TqcF1tGi7X0KdX6Q12sb5T0ia79cknrVetZkt4q6T/t/B9Rs2D0m8Xe/xm4urtj/XOVsqRrv1Wl/Gf1/o/wFSpliUr5skr5q+7fD3f9FnV3wXerlM9KKkOO5N2SPtd9vl7SO1RKu63er1XvVO+CI8Ppkm6RJNV6r6QTVMrRQ45xNPaVWJeyv6RPSfqXYY1dsZa+rmEsiMfFvh7rWl/sWzpE0p6xRd7b4jw1J0v6i+7zzZLeJ0mq9WXV+k31To57ln091nOJvS3Wrn70+r29W0/ddt4jae7UD4lYT5K9KdalHCHpLeq9bkCq9VXV+nz3138m6UrV+kr3t83df7d3rzGQei9e57w4U5xrfVK13tF9fkm9G/Jjw/4USf9A0v/o+nFebBkt1lPXiSrpsG5bh0raqt5N5+wYw8ssa5U+2H3+RJV+u/t8a5V+p6/fH1bpR7rPx1fpnu7zb1bpE93nd3bbW9wtf6NKa82/HzXjWFel5X3LD+7czmC/66t0jnnp5rVVuq9K363S1VWa37X/+ypd3X0+v0qvVemcPfTi0H0l1ldU6V90n7eFcV/RfX5vN75FxHo3xLq3fHm3vceqdBJxTsX54SrdUaXvVOmf9rX/ZZXe033+hSq9FNb7xzv3bU/929djLZ1QpZerdGeV/k+VLibWI9QPaXGVNvT1Oa5K6/qW93z9INbEeqpYS2dX6fbau7a7s0qfrdIh3d/WVulXq/Strlac17feBVW6u0rbqvSTxDmR07v+fkKVHq3S4aH9LVX6tunPeXGcsZb+XVcj1lVpSdd2WJW+XqUnu5x+5zBxGkegX6/SvO7zqiqt7Qv03+7rtzkE6/EqHdp9XtXXb+u0AZp6HJni8a6dB7+9qF5apVKl+VX6XN/BP7xKv9+N8/NV+qsqnb2Hknrvj7W0rErf7NuP/pvFZVW6ofYK+zVV2lSlBcR6N8R6cP0PVOlzxDl1oXds99+jqnRXld7SLZ9apa/W3o3NL1fp2bDeXDgp7tux7tXuRd3nc2rvpHn4rMf3Ny3WU9eP6W9gdrXvufpBrIn11LE+t/b+5/4F3fI1Vfq1vvV/q/au+c6vvf8xVcL6p9XezeZBxHmaOO/626FdTX6v+dvvVukXTTvnxXHHuvf3j1fpV7vPl9beD2ClSid2uT7r8+LueA639n1+ue/zfpIuVK2DPztP9/RRKd+QdJj5y0dV69dC2+OSjpO0Sb3ni4+Q9Gzo88OSfkKl/Lh6jxgcrlL+QLV+SLU+2fV5RaX8vnY8S937afdnuvEUSQ9LemjqQU+UvS/WvccQTpS0oRvPm1TKBtV6omp9QtJ7u/EcKul92vXYyJ5m34r1IF+U9LtTD3iizOU4S7U+3v13s0r5iqTzJf2Feo+o/1j3vSer95jZXGffinXvkbIdj5V9R6U8qN4jq9/Wnmcux3qq+vHTkhaolHnqPZq3vNteZC7VD4lYT5K5HOtNkjap1m91y9dL+ljf325QrVXS7SrlDUmLJT2za8/qPeqZwp2pPV9D5nKcpVIOUE+T+AXVekP42zz1ruvOmXpQc4q9N9a7+IKkP5P0y+rdv1zZ5foGlfKwpFMl3T71wFvGYXCzn6RLu88fkPTNKfp9VdLP71za4Tba0558oGv7u+o34qj1Yu0yK+j/F4MsSX8i6bLu86WSbumCs4taP65al6vWEyT9VNfnQ913L+3+W9TTCqzrlhdolxj0n6h3sfKi9gx7f6xr/V+q9RjVekL3t+07b15KWaxSduTkxyX9t+mCsZvZ12N9Ut8W3inpgakCsZvZe+JcyiEq5bCdn3s3LDvqxFHdf/eT9G8kzUUnzn071j1Nyf7d51WSTtKe+x97e0+sp64fVT3t+I79uEzS/+zGNFfqh0SsJ8neFOunJD2mUk7pWt4haX33+Y8lva0bx8mSDpS0RT1H2nld+wr1LqofmWIfdyd7T5x718y/J+ke1erMl35U0r2qddMU+7Cn2TdiPVgn3i3p3u7zo+rlvtTzWzlFw5wXx/AT7rYqXdX9hHpL3fWc7K1VOrev3+IqXVd7msD1Vfp0176o9h4purtK/7VKG4f8CfegKv1RlTZ0jw6s6tqXVenPTP9dj4D0lm+p0l93+/EHVTq0a7+oSvfXnp7xhiot3IM/l+8bsY77tOvzpVV6oIv3Z+sO3Six3h2xvqYb29rae579DOI8Q5x7j6jc1f27u0r/um/9K7q8vb9KV9b+R5qkR2rv0ZRttfdo9enEejfEWnpfX07fUaW/t0fivLfFerB/PC+u6tbb0G1nh5Z/btQPYk2sp4t1T7f47W4cf1x3XL9JB9bedd66rla8vWv/6VBD3kOcZ6zVP1J7Or3v1l2PaP543zaurdKHzbY5L44z1tKXu334bpX+tO6ScSzrxrfj/uZDw8Sp1FpnfYMZ7ma3qdZDR9sIpCDWk4NYTwbiPDmI9eQg1pODWE8OYj0ZiPPkINYpJvOeRQAAAAAAANirGP2XRQAAAAAAANjn4JdFAAAAAAAAaOBmEQAAAAAAABqmfc/itdde2zyjeuCBBw4sH3zwwc16b3rTm5o212+//QbvVQ866KCmz2GHuVeUtLzyyivTbluS5s+fP+N6krR169aB5Zdffrnp8/rrrzdt73rXu6Z54cr03HzzzU2s43hdXA855JCm7YADDpixbf/992/6zJvXpoPr98YbbzRtEfd4s4v19773vVn3kaQLL7xw6Fh//vOfHyqvXXxczsZ+cdtTref6RVzsX3vttVS/73//+9MuS9Krr77atF1yySVDx/rGG29sYv3884Ovznz66aeb9bZv3960ufjHOXHEEUc0fY455pimbcWKFU3bwoULB5bd8SjmnUou17dt2zaw/OKL7dt2XK6fc845Q8X6/vvvbwZx5JFHDixn8kvyORDnoKuJ8bhKPjdd7cyMwcUrbt993w9+8IOm7f3vf//QOX3llVfOWKtd/XBz3hH33c1lNxdcW6aPG5c7f8Y4uri6Y3vZZZcNHevPfOYzTazPPPPMgWU3v4fNdZdjbp9cnrl1I+4axLXF73Tf5zj77LOHjvVHPvKRJtYxN1z+uFxx1yWZ/Hfbd9czsZ+rFy+88ELT5q4lMjXEtV111VVDx/prX/taE+sYR5fDhx7a+rC4a+RYMzK5KflYx3G5euTmSOY60R03F+vzzz9/qFjfdNNNM+a0q9UufzPXA+5aIHsNHfu5a6Ds+S2en922XNull146ZZz5ZREAAAAAAAAauFkEAAAAAACABm4WAQAAAAAAoIGbRQAAAAAAAGiYVhHvBMlRAJs1XXFE4awTbzojBSc0dcLcSNYIJIqInYDbmTmMwtKlS5u2KFx2x8OJ4118MmJjJ8R1bRnzACe6deLfaLLijrcT4o6CEzRHMxAnJM8SxffuuLlcdGLvmHvueLtj5OIfj5sblxvDKCxevHjGcThTmozJitTu+5IlS5o+Rx99dNOWMU1w43KxduOKwvdoniPl5mQWV49inBcsWND0cfnkiPPS7bPbn4zRkstVh6vDcR+zpiuj4MaROf+4Oua2FeuTyzm3T27f4/F1dc2ZirjjFsfqzvNurKMQ67LU7lNmLkve5CNzbnGxyOSsO7YON664bsa8aFTceOP3Dmte6Noyc2Yq4vnTfZ8zfnH5nzGUGmetlvzxjLXs8MMPb/pkTRqjwVpmPrsxuH5u7FlDmFi33DzKmjllWLRoUdOWMSPL5nTExTlzXSe188HF1MXGXVfH8WeNI6eDXxYBAAAAAACggZtFAAAAAAAAaOBmEQAAAAAAABqmfRA+82Lh7LP07vnl+Pyt06u5F6u6ccVn6d0zwe55YjeuYTV5o+C0Hu5Z5EhWh5l5obL7vsxLcrMvdc6M1b30fNx6gYyeIqPfmIq4n1u2bGn6RN1W9jszmoKp+sU2NwbXNgour6OG0B3f5557rmnLxMcdI7dPmRdER92HlNe/xX1y+gBX75xOMsPmzZubtqj3cnXT4cb10ksvDSxn647TSsR+blxZHV2mfmQ19FlWrlzZtGW0elndUMwxl78uD50GLJPnbgxO3xo1YFkt5Sicd955TVusH278Lofd+S3Gx2kY3bYcMY+z5wunA8xo5merORpmHFFr7eaSy2sXazc3I1ktbsx1F+usJj9znMZ9DZK9Fo24Y+7OUzGP3X5ntXQxjs6HIOudEfu59capWczoEV3ddHnozmUxNzPnO8nPhRjnrBY7M9ey25oOflkEAAAAAACABm4WAQAAAAAAoIGbRQAAAAAAAGjgZhEAAAAAAAAaplU4OmFmFEU6gapbzxnVRLKmNE7kG8Whzigiu60osM2+hHQUMmJTZ3Tg9skJ8mObE41nRddRfO9ExC4vMi/hdeLmcb+Q+MQTT2za4njd8XX75I5JNGfJGqy4WMdj4tZzL6J3ZgVR0O7MHNzL40fBvVg4GmO4fHX76XIjzt9nn312xj6Sz/8YDzcGZ+qRMWPJzElJOumkk5q2DBlTGtfHjd3V4Siif/HFF2f8vqmIee5MAVybqzMxv9zYnQHEKLixZV4Unx1bzEOXv25bLjfjuq7GuBffZ8iaioxCNLPJfkfG5E1qa7M7Hm5bLtYxL9x6rh46M5uY6+M0/ZiKzAvM3XnF5aI7t8T64M6LWQO3GA93bnZ5Muz5aNxmQkcddVTT5vYhkjXJirhYu5xy24rXDVu3bp2xj+SvS5YtWzawnD3PD4urwzGGri64NncPE9ucMZ8zy3HXl3FuufrqzoGufsT9dmPIGE71wy+LAAAAAAAA0MDNIgAAAAAAADRwswgAAAAAAAAN3CwCAAAAAABAw7TOIU58mhHTOjGlE61GMwonws2amzz22GMzrucE3M7QI+63E2tn4jAbnNg0Cq9dXJ2phDP5ePrppweWn3/++aaP2ycnOI8icSfWdQZDru2www4bWI5GGtLshbgz4fYzHnMnJHYibido3rJly8CyMwNxgnmX/3GsMV6SP5ZOyB/j74TRLsdGwe173CdnGOJi7WL2zDPPDCw7cwJ3vDM55eLj4p8Rq7t5NE4hv9vvJ598cmDZif3duFxN2bhx48ByzHHJH5/McXTHx9UKV6tjnru6P+764Uw+4vc6IwoXVzd347F0c8jljvvOmJsLFixo+mTn2uLFiweWjznmmKaPq5uj4OIT89jlj6vLmzZtatqeeuqpgWUXa1cTncFNPIe4WuHqsmuL67ptZQzjZoOLY8Z0w+WPq+kxZ52JSLZ+ZwyA3LVExjRk3IZ6DlffIlnzws2bNzdt8XrPXcNmvzNj+OiurTM4ox+X68Pizs1x7mavxVxNeeSRR2bs43Dn3RhXV2PcucfF8LjjjhtYdjXGzY/p4JdFAAAAAAAAaOBmEQAAAAAAABq4WQQAAAAAAIAGbhYBAAAAAACgYVolb8bExYldnbg5GlFI0uOPPz6w7MShWaFpFKk6YaszSIhCUElatWrVwPLRRx/d9Bk3zkwoCti3bdvW9HFtUbQvSffff//AsjOocEJ1J0KPuFg70e2JJ57YtMVYO6G0E/qOgsupGGsnCHfGQRlxuYv1Cy+80LQ5M4eMuNzlzooVK5q20047bWD5+OOPb/qMW9zvxONR3J01A4licqk1cXE57EwBXA2J43B57YTi0fhDko499tiBZVdDxmm8cuSRRzZtS5cuHVh2pgou52L+Su3xcMfnwQcfbNpijXe4sTvzFJcnUfDv6s78+fNnHMNscGOLeb5169amT9bIKZqsxByXfN3P4GqpM5RYvnx50xZrkYvruGt1xijF1XN3DnS5GE1vMrkv+fNi3HcXH3cN4mr16tWrZ1zP4WpRFvcd8XyQra+Z86c7n2ZNv+K1qTP+yMYsc85ztWcU3PbiPrlYu7ri8vqJJ54YWM5eb7jtx3G4eu3a3LVcvOaI1yRSe+6UpCVLljRtGdy44jx1uera3LVMjKGrH65Wu7oWjbTc+dqde1wtitcuLn7OwG86+GURAAAAAAAAGrhZBAAAAAAAgAZuFgEAAAAAAKBh2oe13TPf8blqpxdwz/a6Z6bjs9bupblOW+RenBt1UFGvMxWZZ+Tdy2+dNmYU3Djis9XuJZrueWj3DHvUe7k+7nlrpzWNz1KvXLmy6ZN52bTD6e9m+2z1TDh9Q9QQZHUYmZfkPvbYY00fF2uns4l6AZd37kXbmbx2fZyOcRTcy2ejlsdpo51ew7XFY+I0pA8//HDT5vpFjYXTBzj9khtX1DY6/cY4X6rtjmVsczXYaVRcbKIe8b777mv6rF+/vmnLvNzd6XecpsNtK6MlGudLniWvg8noXZ1G1cU61gGneXb11fWL+epqxUknndS0ufFH7WTWF2AUXM2NGh1Xg9150WnkNm7cOLD86KOPNn3cNUjm5duuVrua4sYa99vp/ccdazeXMnpfdy5zcYy56GpP1NpJ/rhFXH11Oi0X/+hb4fTl46zVkj/nxfqQ0chJuVx350DnAeC2Fa+l3TWC0/JntMQuhw8++OCmbVjNoruejLGPWkEp78US70XcXMheV8e8c1pmd+2a8R1wGnQ3Z6aDXxYBAAAAAACggZtFAAAAAAAAaOBmEQAAAAAAABq4WQQAAAAAAICGaQ1u3EshozjUGSs4kawzV4jCdGdc44wUnMnEBRdcMLB80UUXNX2caHXDhg1NWzQkceLaUV5+63AC4Sh4dkJcJ+B1L0aO67oYOlG6E89GQ5tzzz236eNwBgxRiO320Yl6RyEjVnfGLA5nUBFzNvPSXMkL+aPI+bzzzmv6uPyMxg1Sm9fZl6GPQsYIwuWYM/txRkEPPPDAwLKbz+5l8e74xn13AvDsvIkmHM4ExxkpDIszHYj129VqZ0KQMQmKcZe8kN+ZU8WxOoMVlzfOECOaIbi6PG4jkIzxmzvXuH1y/WL9djXRbcvVlJjnznTFvQTbmQLF/HGGDy6fRsHtU4y1MwJx67k68NBDDw0su1rt2lytjiY0bk66uLp5E/fJGaI4A4xRcMYi7nsj7hrEnWPjdaEz+YgmSpKvR3GsrrY546Nh83O2ZiAz4WIW98GZC7l66o5RjLXbb3dOcjGL9WjNmjVNn7POOqtpc+eIaKrjcn+c1yAZcyRXS921hjMEitcbWSMhN65oPuaMI919wl133dW0xePtrvVcfk1ncMgviwAAAAAAANDAzSIAAAAAAAA0cLMIAAAAAAAADdwsAgAAAAAAQMO0BjeOKLx2wuKskUI0i3CiT2eM47YfhfxOtO9MLZx4PQp/t2/f3vQZN87oIIrERzEPqLXO2MfF1Y1r+fLlA8sXXnhh08eZ2bi2+J1OrO3E4KPgcirmhou1E2O7/Il57YybnLjfjeuUU04ZWHamKE707IyhYl47Y4hxG1S4ufPGG28MLDtRvWtz8YmxdaJ9l/su12McnfGKM85wAvMoHo/7LEmHHHJI0zYszmAgGqO4sTuTEhebiMsTN7+d2VYU28d6MtUY3FijYUjGlG1U3PZiPNzcytRgqa0fzmzBGVg5Y7lYL5zBzZlnntm0uZobjSCc8YgzNnGmXFncMY/f4WqMq7nRdMrh5mlmDFKbx29+85ubPs6g5M4772za4ljduKIB3qhEgw2Hy313LnMxizXd5aur+65OnnHGGQPLzgzEna/dWGOuuzg4Y6JRcLUyXms5MzVnAOSMZOJ1ifu+rMFQPJdt2rSp6bN06dKmLWNo6MwLneHdsLgYxnnjarWrH64t1h5XY9y5zJ0L4rqurrm6nLk/cfVjttd6/LIIAAAAAAAADdwsAgAAAAAAQAM3iwAAAAAAANDAzSIAAAAAAAA0TGtw40SRUdjtBJfOSMaJPKMw05mpOAHs1q1bm7YNGzYMLN9yyy1NHydcdgLYJUuWTLssjd90JWMqEQ0rpsKJ72McFy5c2PRxRhMZkfhtt93W9HFCX5cXixYtGlh2BgDOJGMUnAlGHJsT8juRuzOcicJhd9zc3HJC5RgPZ3bh5ojLz2j65Mbu4j8Kxx9//Ix93Bx08XHzPuaZG7/LO2eEE4X8WQG4M1eI5jhr1qxp+ixevDi1/Qxx7FJrFODMI7KmVjGGzlTJ1VdXB9atWzew7IyL3PxzRhdxrG49F5tRcOeDOMfd/HNGCi6O0QDImUe43HzqqaeatoMPPnhg2dX40047rWlzRGMIt4/HHHNMaltZ3LGL5wN3vnN1wJ3z4hx0tdrVD2fgEmMdDYEkPx9cXsfxO8M+lxej4Ay94njd+cfF38U65oYzDHHnWFcnL7744oFlZ9K0fv36pi3WHqmdb/GaRBr/eTFTY13eufng6m7MDVcXncGQOx/E+XbHHXc0fZzRnyOO1Y3d7fewONPMjEGWm9+OTC1ypmLOnCqO9dZbb236uPnhro+XLVs2sJy9J5sOflkEAAAAAACABm4WAQAAAAAAoIGbRQAAAAAAAGiYVrPonnOOWiKn/XHP0Lpn2OPzyk6HceqppzZt7jvjuB555JGmj9NTrFq1qmk7+eSTB5adRmXcL3p2WobMS3fds8hOBxP1ae7ZdxcfpxOL++50je54xxfMS9Lpp58+sOye745akFFxMYs56+LjtBNObxc1WG6f3LP6mWfInc7DaYdWr17dtMXju2LFiqaP0zGOgtOjZV4+7DSLTpsZNQKu9ri8drGO41i7dm3Tx2lXnC7TaYwimZfpZnH6k1hTMrpxyb8UOR4Pl9MOVz+iHsRp+ZxuKGqLXJvTf02iVkec5sW1ZbTRTmPt5ryrk3H7Dz30UNPHaWOc9iZuy+kf3bwdBfcC8DjHnc7Q5bWrnTHX3bF1c97NpfhCc6fld9cSLtfj+dO9xNvVzXETa7Wrwe5clnnpvNtvN0dcLkZ918aNG5s+7nrSXRvFNlfbxn1edMcuXl1eWDYAAAhmSURBVJe4mu7qhYtjzH+3LTd/3TVOzHU3t9zxjro5qdVSuto2Wy3ddLhtxXqRvYdxesEYe5errj5lPD1cjXfxcscx3sO4HJntdTW/LAIAAAAAAEADN4sAAAAAAADQwM0iAAAAAAAANHCzCAAAAAAAAA3TGtxkXuLsBO3uhaNOmBlfBpw1onDfGcWgThzq2pyYOYpwnbHCuF/07ATI8TucyN2JVJ1IOR5LJ9Z14lz3nZkXULu4OiFuFI47AbfLQyeqz+Je1Brj6HLYifudoUbcJ3eM3Pbdvsd+br+zZiAxL5zZwrhfPuyE3LHN7berF84IJdYHZ6zk8tOZHMW64uaDi3XGoMLFdZxmIM4wIZpFuGPhhPzOBCKaRbi5MKzhSdZMauXKlU1b5iXw7uXoo+BeQB3jmDXqcHGMbW4ur1mzpmlzZm1xHO54u/gcd9xxTVus6a5+jPOF2pLP2VhP3T659dw8jbXH1Rh3DnTmc/H4urrv6oA7vjGvM2MfFWcIE2unm6tujrvrtrgtt98u1s6Y6JlnnhlYzr5E3R2TmNfufOGuQUbBmarENnc8MtcIrp/LO1fnM+d/d7xdfrrzQawZrv6Nk0xsXP1w1x/OtC7WO2fol91+jIUbe8ZkR2rrh1tvtkZC/LIIAAAAAAAADdwsAgAAAAAAQAM3iwAAAAAAANDAzSIAAAAAAAA0TGtw48wioujSiXCdyN21RUGnM+9wQnLXLwpnnaDTjcH1i+Mat2jfkTGfcLFwYmwneI3bcoJtty1n4BLbsnF14ukoOHcC7nEagUheXB733Y3D7acz4siItt32XfyjCUAmXyUv5HdtERebUdi+fXvTFvPHmSa4cbi8jttyom0Xa5eLcfsu990xcsS56owbXO0clieeeKJpi2Ydbs47Eb2LTdxW1kDM5VycR66PMwBwuR9xeRMNuUbl0UcfbdqWLVs2sJytpZlzmZvfzogls5/u2DqjGmfmEI1A3DFy56hxE41kXA478w5XZ2J9dUZRDleLYn46QyM3BndMMkYj4zZdcUYc8TucgUfW5DDi5rMzxnP5H+dX1vjD9Yv77ebRuM+LjhjH7DWIy5VosufmqqtRLtYxZ10fN9/c9mNeuNwZ5/Wei1ccl7v2jvGTfP2O13/ZPHHxirXIxc/NUTeuWHtcTGdbq/llEQAAAAAAABq4WQQAAAAAAIAGbhYBAAAAAACggZtFAAAAAAAAaJhWhewE2lHc6kxwXJsTFkdhphMWZ00mYj8n2neiUse2bdsGlp1pxrhNV1588cWmLQpQs8ZBzjQhmkg4Ua8TvLr4xzgOa/ohtUYNr7zyStPHGYFEMfBscOL7eDydgYTD5VnE7ZPDzZF4LDOCbcnvY4y/28eY+1LOGGcqnBg+trm55MwJXFvMDZfXro65ORJNADIxlLwRQRyHM4ZwdWVY3H7H8btxOsG8i000UciK493xz+S0y4mMeZE7Ztm5nOXpp59u2uIccYYSGYMVqc3pzZs3N31czFytjuNwZhiu7rhxRRMftz/uPDYK7jsiLoczJm+un7sGcfXVxTHmXtbgxvWL43BmNqPUZYerUTEX3Tjc+dnN1Ux8nLFIxmzJnYfdMXI5EGunOwc+99xzTdsoOEOyWCtdvXaxdjkbr9FcTXdkDCXdtlzddeZ2mfowzvNixojN1Wp3/nH7HXPHredyzo0rHjNXK7KGQLHNXYO6/JoOflkEAAAAAACABm4WAQAAAAAAoIGbRQAAAAAAAGiYVrO4ZcuWpi0+J599Yb17pnnp0qUDy+5ZZafFcc/yxnG4Z+vds70ZvZ3TvLjnsUfR0bmXecbnjF2sszpD1xZxLw53xy3GzB0PdyxdHON+ZzWLo+DGFseRfam2ex49aggyLxWW8nMp4p49d/sYc9bl8Dj1AlJOY5HVUTliP5eLLv6Z5/Xd8XA6IVejov7H6XOy+5gho8vL6BqlnLbb1SsXr4xWPTOHpJyOMaupGoUHHnhgxu845ZRTmj4uri4vMi8cd7HI5KvbltPsZHLTxXrcL4p3+Rnnroura3Mxi+fsrK43088dW7etjCeCO7aZF9/PBjdPXnrppYHlJ598sunjNLyZl5PHbUv5vI76WadRdXmd0Ze745b1Hcji9KGu5kWy5+wYM3edmD0fxGPpxuBwORDHuruv9zL76I63yxN3zRavN1ycXUwzfggufpn8ldoYum3NNs78sggAAAAAAAAN3CwCAAAAAABAAzeLAAAAAAAA0MDNIgAAAAAAADRMq5B2LyeNL9XMGiRkXobqxM3OVMQJnqNwNiOulXICWydGdiLf1atXN21Z3DjiPmSEzKPghL4ZEb0be8ZgRWoNEdxxy4jlx40zSsmYIbl1nSlK1jQhxt+t54yJXMzi/HLfN+5YO8OLaJ7h5r0zbMmYH7j54PLT5XXMfxefrIFK/E4nQt+6dWvTNizupdFxH7NC+4zJkTNAcfPD5Ws8jtm6445jbMsY/YzKpk2bmrY4XmeukTUFii9/djF0++nyNZ4/3XFz23d5HnM4c2xHxY0jfq/LC/fyb1eHY63OmrVlzEhcLFwOuGPp6kVmW6Pg6lisK0899VTT59lnn23aXPzjtlx8oumh5HMgfueSJUuaPlnjJnedGxm3mdDChQubtozp2rDXaFnjIGc2F/u5cWYNmOI43Dnw+eefb9rGSRxX5rpL8rGP++MMblxssqZukex9TaZP5vv64ZdFAAAAAAAAaOBmEQAAAAAAABq4WQQAAAAAAIAGbhYBAAAAAACgoThzDAAAAAAAAPibDb8sAgAAAAAAQAM3iwAAAAAAANDAzSIAAAAAAAA0cLMIAAAAAAAADdwsAgAAAAAAQAM3iwAAAAAAANDw/wG+T9ofbMt/zAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "most_uncertain = np.argsort(np.abs(probs -0.5))[:10]\n",
    "print ('most_uncertain = ', most_uncertain)\n",
    "plot_test_with_title(most_uncertain, \"Most uncertain predictions\")\n",
    "save_name = f'most_uncertain.png'\n",
    "plt.savefig(save_dir+save_name, bbox_inches='tight', dpi=200)\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h3MFjHdzQ_cd"
   },
   "source": [
    "### Loss and accuracy at training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 317
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1572,
     "status": "ok",
     "timestamp": 1590649679231,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "8hFWDkZQB_bb",
    "outputId": "00d655b1-49c7-4bc7-9e8a-11e80b1ffcbb"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbAAAAEgCAYAAADVKCZpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3gVZd7/8fdJJ5000oCQBEIIKYQuiiKC6AqCIuJiAxRF7O762J6f6BZ19XGta2ERZVfBtkoTFMsqKIL0DiEkQAiQHtLr/P4YiEQSSOXkHD6v6zpXYM6cmXtycD7eM/d8b4thGAYiIiI2xsHaDRAREWkJBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkBZiIiNgkJ2s3oL0EBAQQERFh7WaIiNiU9PR0cnJyrN2MJrHbAIuIiGD9+vXWboaIiE0ZMGCAtZvQZLqEKCIiNkkBJiIiNkkBJiIiNslu74GJiH2oqqoiIyOD8vJyazfFrri5uREeHo6zs7O1m9JiCjAR6dAyMjLw8vIiIiICi8Vi7ebYBcMwyM3NJSMjgx49eli7OS2mS4gi0qGVl5fj7++v8GpDFosFf39/m+/VKsB+Y392MSt3HrN2M0TkFAqvtmcPv1MF2G98uP4Qs97fSE2tYe2miEgHkJubS1JSEklJSQQHBxMWFlb398rKyiZtY+rUqezZs+eM67z++uu8//77bdHk84bugf1GVKAnlTW1ZOSX0t3fw9rNEREr8/f3Z/PmzQDMnj0bT09P/vCHP9RbxzAMDMPAwaHhPsG8efPOup9Zs2a1vrHnGfXAfiMq0Ayt1OxiK7dERDqyffv20adPH6ZMmUJcXBxHjhxhxowZDBgwgLi4OJ5++um6dS+88EI2b95MdXU1vr6+PPLIIyQmJjJ06FCysrIAeOKJJ3jppZfq1n/kkUcYNGgQMTEx/PTTTwCUlJRw7bXX0qdPHyZOnMiAAQPqwvV8pB7Yb0QGeAKwP7uES3tbuTEiUs9TS3awM/N4m26zT6g3T46Na9Fnd+/ezfz58+vKLz377LP4+flRXV3NiBEjmDhxIn369Kn3mcLCQi6++GKeffZZHnzwQd555x0eeeSR07ZtGAbr1q1j8eLFPP3006xYsYJXX32V4OBgPv30U7Zs2UJycnKL2m0v1AP7jc4eLvh5uJCaXWLtpohIBxcVFVWvduCCBQtITk4mOTmZXbt2sXPnztM+06lTJ6644goA+vfvT3p6eoPbvuaaa05bZ/Xq1UyePBmAxMRE4uJaFrz2Qj2wBkQGeOgSokgH1NKeUnvx8Pj1PnlKSgovv/wy69atw9fXlxtvvLHBYeouLi51f3Z0dKS6urrBbbu6up51nfOdemANiAr0ZL96YCLSDMePH8fLywtvb2+OHDnCl19+2eb7GDZsGB999BEA27Zta7CHdz5RD6wBkYEefLi+gsKyKnw62W6ZFRE5d5KTk+nTpw+9e/eme/fuDBs2rM33cc8993DzzTfTp0+fupePj0+b78dWWAzDsMsHngYMGNDi+cC+3nmM2+av57O7LqBft85t3DIRaY5du3YRGxtr7WZ0CNXV1VRXV+Pm5kZKSgqjR48mJSUFJ6eW9UUa+t225tx5rqkH1oDIuqH0JQowEekwiouLGTlyJNXV1RiGwVtvvdXi8LIH5++Rn0FXP3ecHS3s10AOEelAfH192bBhg7Wb0WFoEEcDnB0d6ObnrpGIIiIdmAKsEZEaiSgi0qEpwBoRFehJem4J1TW11m6KiIg0QAHWiMhAD6pqDDLyy6zdFBERaYACrBFRgSdqIuboPpjI+W7EiBGnPZj80ksvMXPmzEY/4+lpnkMyMzOZOHFig+tccsklZx2y/tJLL1FaWlr39yuvvJKCgoKmNt2uKcAaUVeVPkv3wUTOdzfccAMLFy6st2zhwoXccMMNZ/1saGgon3zySYv3/dsA++KLL/D19W3x9uyJAqwRvu4u+Hu4qAcmIkycOJFly5bVTWCZnp5OZmYm/fr1Y+TIkSQnJxMfH8+iRYtO+2x6ejp9+/YFoKysjMmTJxMbG8uECRMoK/v1FsXMmTPrpmJ58sknAXjllVfIzMxkxIgRjBgxAoCIiAhycnIAePHFF+nbty99+/atm4olPT2d2NhYbr/9duLi4hg9enS9/dgTPQd2BpGBHuqBiXQkyx+Bo9vadpvB8XDFs2dcxc/Pj0GDBrF8+XKuvvpqFi5cyKRJk+jUqROfffYZ3t7e5OTkMGTIEMaNG4fFYmlwO2+88Qbu7u7s2rWLrVu31psO5S9/+Qt+fn7U1NQwcuRItm7dyr333suLL77Id999R0BAQL1tbdiwgXnz5rF27VoMw2Dw4MFcfPHFdO7cmZSUFBYsWMCcOXOYNGkSn376KTfeeGPrf1cdjHpgZxAV6KkemIgA9S8jnrx8aBgGjz32GAkJCVx22WUcPnyYY8eONbqNH374oS5IEhISSEhIqHvvo48+Ijk5mX79+rFjx46zFupdvXo1EyZMwMPDA09PT6655hpWrVoFQI8ePUhKSgLOPGWLrVMP7AwiAz3I+aWSwtIqfNxV1FfE6s7SU2pPV199NQ888AAbN26ktLSU/v378+6775Kdnc2GDRtwdnYmIiKiwSlUziYtLY0XXniBX375hc6dO3Prrbe2aDsnnZyKBczpWOz1EqJ6YGdwcnbmVPXCRM57np6ejBgxgmnTptUN3igsLCQoKAhnZ2e+++47Dhw4cMZtDB8+nA8++ACA7du3s3XrVsCcisXDwwMfHx+OHTvG8uXL6z7j5eVFUVHRadu66KKL+PzzzyktLaWkpITPPvuMiy66qK0O1yaoB3YGUUEnAiyrmGQV9RU5791www1MmDCh7lLilClTGDt2LPHx8QwYMIDevXuf8fMzZ85k6tSpxMbGEhsbS//+/QFzduV+/frRu3dvunbtWm8qlhkzZjBmzBhCQ0P57rvv6pYnJydz6623MmjQIABuu+02+vXrZ7eXCxui6VTOoLqmltj/t4LbLorkf8ac+R+miLQPTafSfmx9OhVdQjwDJ0cHuvt7qCq9iEgHZFMBtn//fqZPn97oU+3tITLAg1QV9RUR6XAaDbA9e/aQlJRU9/L29q57UK65pk2bRlBQUN3DfKdasWIFMTExREdH8+yzZx5hFBkZydy5c1vUhpaKCvLkgIr6ioh0OI0O4oiJiWHz5s0A1NTUEBYWxoQJE+qtk5WVRadOnfDy8qpbtm/fPqKjo+utd+utt3L33Xdz880311teU1PDrFmzWLlyJeHh4QwcOJBx48ZRU1PDo48+Wm/dd955h6CgoJYdZStEBphFfQ/ll9EjwOOc719EwDCMRh8Olpaxh+EPTbqE+M033xAVFUX37t3rLf/+++8ZP348FRUVAMyZM4d77rnntM8PHz4cPz+/05avW7eO6OhoIiMjcXFxYfLkySxatIj4+HiWLl1a72WN8IJfRyLqPpiIdbi5uZGbm2sXJ9yOwjAMcnNzcXNzs3ZTWqVJw+gbK1p53XXXkZaWxvXXX891113HO++8w8qVK5u888OHD9O1a9e6v4eHh7N27dpG18/NzeXxxx9n06ZNPPPMM6f10gCWLFnCkiVLKCwsbHI7ziTq5LNg2cWMjO3SJtsUkaYLDw8nIyOD7OxsazfFrri5uREeHm7tZrTKWQOssrKSxYsX88wzzzT4/sMPP8zkyZOZOXMmqampdVMItAd/f3/efPPNM64zduxYxo4dy4ABA9pknz7uzmZRXw3kELEKZ2dnevToYe1mSAd01kuIy5cvJzk5mS5dGu59rFq1iu3btzNhwgSeeuqpZu08LCyMQ4cO1f09IyODsLCwZm3jXIgK9CRVlxBFRDqUswbYggULGp3zZtOmTcyYMYNFixYxb948cnNzeeKJJ5q884EDB5KSkkJaWhqVlZUsXLiQcePGNb3150hkoId6YCIiHcwZA6ykpISVK1dyzTXXNPh+aWkpH330EVFRUTg4ODB//vzTBnqAWX5l6NCh7Nmzh/Dw8Lqh8E5OTrz22mtcfvnlxMbGMmnSJOLi4trgsNpWVKAnuSWVFJRWWrspIiJygkpJNcE3u44x/b31fDrzAvp3V01EEbFfKiVlZ6ICNZReRKSjUYA1QXjnTjg7WlRSSkSkA1GANYGK+oqIdDwKsCaKCvTQUHoRkQ5EAdZEkYGeHMwrpUpFfUVEOgQFWBNFBXqaRX3zSq3dFBERQQHWZJGBZiV6PdAsItIxKMCa6NSiviIiYn0KsCbycXcmwFNFfUVEOgoFWDNEqqiviEiHoQBrhqhAD/bnqAcmItIRKMCaITLAk7ySSvJLVNRXRMTaFGDNEBV0YiRiji4jiohYmwKsGSLrRiLqMqKIiLUpwJohvHMnXBwdNJBDRKQDUIA1g1nU111D6UVEOgAFWDNFaSi9iEiHoABrpshADw7mqqiviIi1KcCaKSrQk+pag4Mq6isiYlUKsGZSUV8RkY5BAdZMkYHmUHrNziwiYl0KsN/a9zV8+Xijb/t0cibA01UDOURErEwB9ltHt8Ga1yBzU6OrRAZ66BKiiIiVKcB+a8A0cPWG1S81uoqG0ouIWJ8C7LfcfGDgdNi5CHJTG1wlKtCD/NIq8lTUV0TEahRgDRk8Exxd4MeXG3w7SgM5RESsTgHWEK8u0O9G2LIAjh857W0NpRcRsT4FWGMuuAdqq+Hnf5z2VnhndxX1FRGxMgVYY/x6QNw1sP4dKMuv95ajg4WIAHdNqyIiYkUKsDO58H6oLIZf5p72VmSAp+6BiYhYkQLsTILjIXoU/PwGVJXVeysqyIODeSrqKyJiLQqws7nwASjNgU3/rrc4MkBFfUVErEkBdjbdL4DwQfDTK1BTXbc4KsgcSp+apcuIIiLWoAA7G4vF7IUVHIQd/6lbHBnogcUCP6XmWrFxIiLnLwVYU/QaA4GxsPrvYBgAeLs5M6l/V/798wH2qRcmInLOKcCawsHBHJGYtRNSvqpb/McxMXRyduSpJTswTgSbiIicGwqwpup7Lfh0NXthJwR4unL/qF6sSsnh611ZVmyciMj5RwHWVI7OZnWOg2vgwJq6xTcP7U7PIE/+tHQn5VU1VmygiMj5RQHWHP1uAnd/+PHXqVacHR14cmwcB/NK+eeq/VZsnIjI+UUB1hwu7jD4Tti7Ao7tqFt8Yc8AxsQF8/p3qWQWlJ1hAyIi0lYUYM018DZw9jhtqpXHfxdLrWHw1y92WalhIiLnFwVYc7n7wYCpsO0TyD9Qt7irnzt3XhzF0q1H+Hm/ng0TEWlvCrCWGHIXWBxgzWv1Fs+8JIow307MXryDatVIFBFpVwqwlvAJg8TJsHE+FGfXLXZzduSJ38Wy+2gRH6w7aMUGiojYPwVYSw27D6rLzVmbTzGmbzAXRPnzf1/tJa+k0kqNExGxfwqwlgroCb7d4fCGeostFguzx8VRXFHNC1/tsVLjRETsnwKsNUKT4Mjm0xb36uLFzUO7s2DdQbYfLrRCw0RE7J8CrDVCEiE/HcryT3vr/st64efuwuzFqpMoItIeFGCtEZJk/jyy9bS3fDo58/CYGNYfyGfR5sxz3DAREfunAGuNugA7/TIiwHX9u5IQ7sNfv9hFcUV1g+uIiEjLKMBaw8PfrFB/ZEuDbzs4mAM6sooqeP27fee4cSIi9k0B1lohiZDZcA8MILlbZ67pF8bc1WkUlVedw4aJiNg3BVhrhSRBXiqUNz7acMqQ7lRW17Jy57Fz2DAREfumAGut0BP3wY5ua3SV5G6+hPl2YskWDeYQEWkrCrDWCkk0f57hMqLFYuGqxBBWpeSQr+ocIiJtQgHWWp5B4BXa6ECOk8YmhFJda7Bix9Fz1DAREfumAGsLjVTkOFVcqDc9AjxYulWXEUVE2oICrC2EJEFOClQUNbqKxWJhbEIIa1JzySoqP4eNExGxTwqwthCSCBhwdPsZVxubGEqtAcu36TKiiEhrKcDaQuiZK3Kc1LOLFzFdvDQaUUSkDSjA2oJXMHh2OetADoCxiSGsP5BPZkHZOWiYiIj9UoC1lZCkMw6lP+mqhFAAlm090t4tEhGxawqwthKaBDl7oLLkjKtFBHgQH+bDEo1GFBFpFQVYWwlJBKMWju0466pjE0PYmlHIgdwzh52IiDROAdZWTk6t0oTLiL87cRlxqS4jioi0mAKsrXiHgnvAWUciAoT5dmJA984ajSgi0goKsLZisZyoyHH2kYgAVyWEsPtoESnHGn/4WUREGqcAa0shSZC1C6rOPkT+yoQQHCywRJcRRURaRAHWlkISwaiBYzvPumqQlxtDIv1ZuiUTwzDOQeNEROyLAqwt1VXk2NSk1ccmhrI/p4QdmcfbsVEiIvZJAdaWfLpCp85NGokIMCYuGCcHi54JExFpAQVYW7JYzPtgTRzI0dnDhQt7BrB0yxFdRhQRaSYFWFsLPTGQo7qiSauPTQjlcEEZmw4VtHPDRETsiwKsrYUkQm0VZJ19IAfAqLguuDg56JkwEZFmUoC1tWZU5ADwdnPmkl6BLNt6hJpaXUYUEWkqBVhb6xwBbj5Nqshx0tjEULKKKvglPa/92iUiYmcUYG3NYjEvIzZxIAfAyNggOjk76jKiiEgzKMDaQ0iSWZW+urJJq7u7ODEyNojl249SXVPbzo0TEbEPCrD2EJIINZWQvavJHxmbGEpeSSU/pea2Y8NEROyHAqw9hPYzfzbjMuLFvQLxcnXSZUQRkSZSgLWHzj3A1bvJIxEB3JwdGR0XzIodR6mormnHxomI2AcFWHtwcIDghGb1wACuSgyhqLya/+7JbqeGiYjYDwVYewlNgmPboaa6yR+5MDqAUB83Xv9uH7V6JkxE5IwUYO0lJBGqyyF7d5M/4uzowEOjY9iaUciybZonTETkTBRg7eVkRY5mXkYc3y+M3sFePP/lHiqrNaReRKQxCrD24h8NLp7NqsgB4Ohg4ZErenMwr5T31x5op8aJiNg+BVh7cXCA4PhmjUQ86eJegQyL9ueVb1I4Xl7VDo0TEbF9NhVg+/fvZ/r06UycONHaTWmakCQ4ug1qmzcs3mKx8MiYWPJLq3jr+9R2apyIiG07Y4AVFBQwceJEevfuTWxsLGvWrGnRTqZNm0ZQUBB9+/Y97b0VK1YQExNDdHQ0zz777Bm3ExkZydy5c1vUBqsISYTqMsjZ2+yPxof7cHVSKHNXp3G0sLwdGiciYtvOGGD33XcfY8aMYffu3WzZsoXY2Nh672dlZVFUVFRv2b59+07bzq233sqKFStOW15TU8OsWbNYvnw5O3fuZMGCBezcuZNt27Zx1VVX1XtlZWW15PisK7RlAzlO+sPoGGpr4e8rmx+AIiL2rtEAKyws5IcffmD69OkAuLi44OvrW2+d77//nvHjx1NRYc4+PGfOHO65557TtjV8+HD8/PxOW75u3Tqio6OJjIzExcWFyZMns2jRIuLj41m6dGm9V1BQUKsO1CoCeoFTpxbdBwPo6ufOTUO78/GGQ+w9VnT2D4iInEcaDbC0tDQCAwOZOnUq/fr147bbbqOkpKTeOtdddx2XX345119/Pe+//z7vvPMOH3/8cZN3fvjwYbp27Vr39/DwcA4fPtzo+rm5udx5551s2rSJZ555psF1lixZwowZMygsLGxyO9qNg6M5kKOZIxFPdfeIaDxcnXhuedOfJxMROR80GmDV1dVs3LiRmTNnsmnTJjw8PBq8R/Xwww/j5ubGzJkzWbx4MZ6enu3WWH9/f958801SU1N59NFHG1xn7NixvP322/j4+LRbO5olNAmObIXalj3T1dnDhbsuieab3Vn8vF+V6kVETmo0wMLDwwkPD2fw4MEATJw4kY0bN5623qpVq9i+fTsTJkzgqaeeatbOw8LCOHToUN3fMzIyCAsLa9Y2OryQJKgqgdzT7w021dRhEYT4uPHM8t0YhkpMiYjAGQIsODiYrl27smfPHgC++eYb+vTpU2+dTZs2MWPGDBYtWsS8efPIzc3liSeeaPLOBw4cSEpKCmlpaVRWVrJw4ULGjRvXwkPpoEISzZ8tHMgBZqX6B0b1YsuhAr7YdrSNGiYiYtvOOArx1VdfZcqUKSQkJLB582Yee+yxeu+Xlpby0UcfERUVhYODA/Pnz6d79+6nbeeGG25g6NCh7Nmzh/Dw8Lqh8E5OTrz22mtcfvnlxMbGMmnSJOLi4trw8DqAwN7g5Naq+2AA1yaHE9PFi799uVslpkREAIthp9ekBgwYwPr1663dDNOcS83RiFOXtWoz3+3OYuq7v/DUuDhuuSCibdomInKKDnXuPAubqsRhs0L7QeZGKG7dPF+XxAQyNNKfl79JoUglpkTkPKcAOxcGzYCaKljxSKs2Y7FYePTK3uSVVPL2D/vbqHEiIrZJAXYuBMbARQ/B9k8gZWWrNpUQ7svYxFDmrNrPseMqMSUi5y8F2Lly0YMQEANLH4CK4lZt6o+jY6ipNVRiSkTOawqwc8XJFca+DIWH4Lu/tmpT3fzduXloBAt/OcTSrZlt1EAREduiADuXug+FAdNg7RtweEOrNvXwmBgGdO/MQx9tYfOhgjZqoIiI7VCAnWuXzQaPIFh8nzmwo4VcnRx566b+BHm7ctt76zlcUNZmTRQRsQUKsHPNzQeufB6ObYM1r7VqU/6errxzy0AqqmqY/u4vFFdUt1EjRUQ6PgWYNfQZB72vgv8+C7mtm3G5ZxcvXpuSTEpWMfcu2ERNrV0+ly4ichoFmLVc+Tw4upijEltZDOXiXoHMHtuHb3dn8dcvdrVRA0VEOjYFmLV4h8JlT0La97BlQas3d9PQCG69IIK5q9N4f+2BNmigiEjHpgCzpv7ToOsQ+PKxVpeZAnjid7FcEhPI/1u0g9UpOW3QQBGRjksBZk0ODuazYRXF8GXDE3Q2h5OjA6/e0I/oQE9mvr+BfVmte2BaRKQjU4BZW1Bvs8zUto9bXWYKwMvNmX/eMgBXJwemvfsLeSWVbdBIEZGORwHWEdSVmXqw1WWmALr6ufPWTQM4erycO/+1gYrqmjZopIhIx6IA6wjqykwdbHWZqZP6d+/MC9clsi49j8f+sx07nfZNRM5jTtZugJzQfSj0n2qWmep1OURe3OpNjksMZX92MS99ncKP+3LoFexFTBdPenbxolcXL3oGeeLhqn8CImKbdPbqSEY9Bemr4F8T4NInYNj95kCPVrhvZE8CvVxZn57P3mNFvLc/l8rq2rr3wzt3oteJQOvVxZPkbp2JCPBo7ZGIiLQ7i2Gn15ZsaVrsesoLYcl9sOMziLoUJrwNnoFttvmaWoODeaXsPVbE3qNF7M0qJuVYEanZxVTVGDhYYOqwHjw4qpd6ZyLnIVs6dyrAOiLDgA3vmjM4u/nCtf+EHhe16y6rampJzylh3k/pfLD2ICE+bsweF8flccHtul8R6Vhs6dypQRwdkcUCA6bCbd+AqxfMH2fWTaxtv9GEzo4O9OzixV8nxPPpzAvw6eTMHf/aoEr3ItJhKcA6suC+MOO/kHA9/PcZmH81HD/S7rvt370zS+65kEev6M2P+3IY9eL3zPlhP9U1tWf/sIjIOaIA6+hcPWHCmzD+xCSYb14I+75u9906Ozpwx8VRfPXAcIZE+vOXL3Yx9rUf2XQwv933LSLSFAowW5H0e7M35tkF/n0tfD27VRNiNlVXP3fm3jKAN6Ykk1dSwTVv/MQTn2+jsKz99y0iciYKMFsSGAO3f2M+L7b67zB3NOz/vtXTsZyNxWLhivgQvn7wYm4ZGsEHaw9y2Yvf8+WOo+26XxGRM1GA2RrnTjD2JZg4D4qOmAM83r0K0le3+6693JyZPS6Oz2cNI9DTlTv+tYGHPtrC8XL1xkTk3FOA2aq+18C9m+GKv0FuCrz7O3hvLBxY0+67Tgj35fNZw7h7RDSfbcrgipdWsSY1t933KyJyKgWYLXN2g8F3wH1b4PJnIGs3zBsD88fDoXXtumsXJwf+cHkMn8y8AGdHCzfM+Zk/Ld1JeZUKB4vIuaEAswfOnWDoXWaQjf4zHN0Gc0eZgz0yNrTrrpO7deaL+y7ixiHdmLs6jbGvrmb74cJ23aeICKgSh32qLIF1c+DHl6EsD3qOhoiLwCPwxMvf/OkeYPbi2sj3e7P548dbyCup5L6RPZl5SRROjvp/JBFbYkvnTgWYPasohnVvw5rXoLSRe1QuXuARcOIVCEGxMGSWGXItUFBayROfb2fp1iMkdfXlxUmJRAZ6nrZeUXkVx46Xc6TQfB0rLCe3pJIr40MY1MOvRfsWkdazpXOnAux8YBhQWQwl2VCSc+KVbb5Kc3/9c0kOZO0EF08Ydh8MuQtc3Fu0y8VbMnnis21U1tQyeWA3iiuq6wLraGE5xRXVp33GxcmByupark0O59ErexPg6draIxeRZrKlc6cCTOrL3gNfPwV7loFXCIx4DBJ/D47Nr0x/tLCcR/6zlVUp2QR5udHF240Qn19/Bvu4EeztRohPJ4K8Xak1DF77dh9zVu2nk7MjD4/pzQ2DuuHoYGmHAxWRhtjSuVMBJg07sAZW/i9k/AKBveGy2dBrjFlo+GyqK+HgT7D3S9j7JcbxTCxd4iAkAUISITgBgvo0ev9tX1YR//v5DtbszyUx3Ic/j48nPtynTQ9PRBpmS+dOBZg0zjBg1xL45inI3QfdLoDRf4LwAaevW5wFKSth7wpI/Q4qi8DR1ZwGxi8Kju2Ao1uh4ri5voMTBMSYgRaSYIZacDy4eZ/YtcHiLZn8aekucksquGlIdx4aHYNPJ+czNrm21mDPsSJ+3JfD6n05bD9cyIR+YTw8pjfOGlAicla2dO5UgMnZ1VTBxvnmlC4lWdDnarj0/5n31fZ+CSlfmoWGwYSxyWYAABm+SURBVLzs2Oty6Hk5RF4MLqfM7lxbCwXpcGSrGWZHtph/Lsk6sYIF+k2BUX8Cd3MgR2FZFX9fuZf5a9Lx83Dh8d/FMj4pDMspPcGM/NITgZXLmtQccoorAYgM8KC7vzvf7cmmXzdfXvt9MmG+ndr/9yViw2zp3KkAk6arKDZHNP74ClSVnFhoMXtkPS83gys4vmmXGU9VdNQMs9RvzeH/7n4w5lnoe23dtrYfLuTxz7ez5VABg3v4cf3Arqw/kM+P+3I4kFsKQKCXKxdGB3BBlD/DogMIPRFWy7Ye4X8+3Yqjg4UXJyUyMrZLW/1GROyOLZ07FWDSfMVZsPE98A6HnqPMIfht5chWWHIvZG4yn1/73Yvg2xUwLw8u/OUQz63YTWFZFZ6uTgyJ9GNYdADDogPoGeRZr2d2qvScEu56fyM7jxznjuGR/OHyGF1SFGmALZ07FWDS8dTWwNq34Ns/m38f+b8waAY4OALms2aH8sroHeLVrBAqr6rhT0t38v7ag/Tv3pnXft+PEB9dUhQ5lS2dOxVg0nEVHISlD8K+lRCaDONeMS9RttKizYd57D/bcHFy4MXrkxgRE3TWz+QWV/BTai6rU3LYcDCfrp07MaiHP4N6dCY+zBcXJ/XmxD7Y0rlTASYdm2HA9k9hxSNQmgfD7oWL/8es/9gKqdnFzHp/I7uPFnHXJVE8OKpXvbJX5VU1rEvLqxvNuCPTHD3p7eZE/+6dycgvIyWrGAA3Zwf6de3MoB5+DO7hR79unenk4tiq9olYiy2dOxVgYhtK88zn0jb9Gzr3gKtehKhLW7XJ8qoanlqygwXrDjEowo/7R/Vk86ECVqfksP5APpXVtTg7WujfvTMXRgdwYc9A4sN86h6szi2u4Jf0fNal5bEuPZedmcepNcDJwUJ8uA+DevgxKrYLAyJUGktshy2dOxVgYlvSfoAl90NeqjnycdTTENS7VZv8fNNhHvtsG6WV5lQwvYO9uKinOTBkUA8/3F2aVoXkeHkVGw7k80taHuvS8tiSUUBVjcHkgV15/HexeLmd+Rk2kY7Als6dCjCxPVXlsO4t+OH/zAem+91klrzyCm7Z9kpyyV8zn6LDu/HrFotnWB8I6Am+3eoGjrREaWU1r3yzj7d/SCXY243nJiZwUc/AFm9P5FywpXOnAkxsV0ku/PA8/DLHrPox7F4Yeje4nl79/jS1tZD2X/MB7d3LoKYSXL1/rRQC5jb9o80wC+h14tXTXNaUfZyw6WA+f/h4C6nZJfx+cDceuzIWT1cnqCqD45lQdMS8RNrjIujUufm/h98wDIPckkqOFpYTE9y8kZrSCMOA/f81K8e42/clYVs6dyrAxPblpprlrnYuAs8uZm8s6caGCxAXHobNH8Cm+eYox06dIWEyJN8EXeLMUMzZC7kp5s+cEz/z08Go/XU7Lp7QyQ86+ZrbcPczf3bqfGL5iWVYoCiT6oLD7Ni9h4KsA4Q7FtDNuRDnyt9M/OkeYF4STbwBHM4eOuVVNaTllJCWU8L+7GL2Z5eQmlNCWnYxx8vNav99Qrx54bpE+oR6t/z3K7Dq/+Cbp8HZw/y3MmQmdI6wdqvahS2dOxVgYj8OrYOvnoBDayEw1gyDnqOgttosebVxvjkk36iFHsMh+RbofVXTJvWsroC8/SfCbZ859UxZvtlzKss3Jw4tyzdfpwZdHQt4BlHiGsSmAnf2V3gT0rUHw5MTcPULB4sDfPsns3hy1yHwuxfqPTKQXVTBxoP5bDyYz87M4+zPLiGzsIxT/+sN8XEjMtCDHgEeRAZ44urswN9XplBQWsndl0Zz1yXR52a4f3UlbPqXWWrsgnubX5mlo9m5GD66CWLHmvPnbfsYjBroM97s9Yf2s3YL25QtnTsVYGJfDAN2LYavZ5uBEz4ICg5A8THwDDZrLfa7Efwi22f/tbXmZciToWYY5r05zy7gaA7iKK+q4YUv9zD3xzTCO3fi+YmJDIn0Nz+7+X2Mr5+Esnx2dZ3MfNcb+PFwNYfyygBwdrQQE+xFVKAnkQGevwZWoEeDg03ySyp5askOPt+cSWyINy9cl0BcaDtV9q+tga0fwX//avZuAQbdAVc8Z7shlrkJ3rnC/J+JW5aY/7NzPBPWvgnr55nfdcRFZlBHX9aknnNHZ0vnTgWY2KfqStgwD37+h9kb638LRI9q0bxm7eWX9Dz++PEW0nNLmTK4G15uzmw8mM+BjAxmGR9yo+PX5Ft8WBx0J9Vxk0iO8CMu1Bs357MMLCnJgYNrIGsXxF8Hfj34asdRHv98O/klldw1Ipq7R7Rhb+zkrAXf/QWyd5szC4x8EvZ/Z9bOHHynWduyFSF2KK+Uv6/cS3l1DS9OSjr776AtHM+EOZeCgzPc/g14/uaB9/LjZkm1n9+A44fNaYcuuMf8nTvZ7mSstnTuVICJWFFZZQ1/+3I37/6UjpODhT6hPiR38yW5W2cGux0k8IfHsRxeD92GwpUvQHDf0zdScAgO/GTOwXbgJ/My50kegTDlEwhNoqC0kqeW7OSzTYfpHezFC9cl0jesFb0xwzBD6punzZ5KQC8Y8TjEjjN7IoYBXz4OP78Og2fCmGeaHWJ5JZW8+m0K//75AA4WCxXVtYyJC+b1KcntO9FpZQnMu8K8vzr9K/P+aGNqqsyH7X96FY5tN3v6Fz0Eg263yZ6nLZ07FWAiHUB2UQVebk6n9yxqa2Hzv2Hlk1BeCIPvMAd5HN5g9rIO/ASFh8x1Xb2h2xAz7LpfYA40WTAZygpg8vvm9DbA1zuP8dhn28gtqeSuS6K4+9JoXJ2a2aM5tM4MrvRV4NMVLnkUEq4/vYdrGPDlY2ZPeMhdcPlfm3RSL6us4Z0f03jzv6mUVFYzaUBX7r+sF8u2HeFPS3cyZXA3/jy+b6PFm1ultta857XnC7hhoTnLQlMYhjmjwuq/m7+XPlfD1f9o1ojVjsCWzp0KMBFbUJpnDvJYPw848Z+sR6AZVN0uMH92iTv9ubXCw/Dva80Hv6+ZA3HjzcWlVTy1dAf/2XiYmC5ePDwmhmAfN7zdnPFxd8bTxQmHhno4R7ebRZb3Ljf3P/yP0P/WM18yMwxY8SisfcN8zGH0nxsNseqaWj7ekMHfV+4lq6iCy2K78D9jYujZxatunWeW7+Kt7/fzwGW9uO+yns34JTbR17PNEBrzrDnasLkMA356xdxOQIz5Pw/+UW3dynZjS+dOBZiILTmyxZzdOnyQeVJsSg+kNM/siR1aB7/7Pxg4ve6tb3aZvbFjxyvqfcTBAl5uzvh0csa7kxPhLiXcWDKfC45/QbWTFxWD78br4rvrT1h6JoYBy//HfAC9gRAzDIOvdh7jbyt2k5pdQnI3Xx69MpaBDZThMgyDhz7ewn82HuavE+L5/eBu9VfITYXDG6H7UPAJb1r7Ttr8AXw+E/pPhav+3uxLgIZhMO/HdI4VlfPH6Eyc/jPdHJV6zT+h1+jmtcVKbOnc2XHuaIvI2YUkmq/mcPeDmz6Hj2+FZQ9CSbZZENliYWRsF76J9GdrRgHHy6o5XlbF8fIqCsvMV3FpGQOyPuXqo/NxNcp5nyt5vvhqjn/jSa/t6xneM5DhvQIZ1MPvzAMrLBZzNCKGObDD4gCjnqasqpZNh/J58au9rD+QT2SgB2/e2J/L47o0ennQYrHw3LUJ5JVU8sTn2+jiWsFI193m5bvUb81n9gAsjhB7lTmIpNvQs4fRgZ9g8b3Q42K48vlmh1d5VQ0Pf7KVxVsyAUjNCuL1ad/i+snN8MEkuPRxuPAhuxip2FGoByZyvqipMk/QWz6AAdPNk/SZSmWlfgvLH4GcPRA1EsY8gxHQi91Hi/hhbzY/pGTzS1o+lTW1uDo5MDjSn+E9A7gkJpCowF8nFy2rrOFAXgnpOeYrceufGZr3Gf9yGM//ll4HWAj0cuWBy3oxaUB4vVkBGlRbA5mbqNyzkn1rFtOrajdOllrznl+P4WaR59B+5uMUG96D8gJzZOTgO81Zvht67i9vP8wZaYb9bV83uyLKkcIyZszfwPbMQv4wOgYvNyeeXLyDQRF+zPl9H7y/+gNs+8h87nD8G+DWcR8st6VzpwJM5HxiGPD1k/Djy+Ygg2vmnH7/Ki/NfCB891Kz2sSYZ6HXmAZ7JKWV1azdn8f3JwJtf3YJAKE+bnT1c+dAbilHj5fX+4y/uzN/cX2XMWXLWB9+K0cHPsylsV0aLppcUWz2qPL2Q36aeWlw/3/NUMJCdXASC/Oi+bqyL4/efjMxYb+55FhZagbHz29C9i5w9zcvDw6cDt6h5jplBTB3NJRkwW3fNPt+1YYD+dzxrw2UV9Xw0vVJXNanC2DOO/fQR1uICfbivakDCdj+jvl79Y+CyR+YZck6IFs6dyrARM5HP71qnkx7DIfr3zd7BJUlsOpF8z0HJxj+EAyZ1bRKJSdk5Jfyw94cftibTXZxBd393Ynw9yAiwIMe/h50D3DH283ZHOn3xUOw/h0Ydr/ZMzkZUnlpv/4syaq/A+8wiBph9rIiR4C7Hxn5pVzzj59wsFj49K4LCPNtYK44wzBnMlj7ljm60MHRHO4/aAb88DfzvZs+N+tRNsNH6w/xxGfbCfF14583D6g32ATguz1ZzPz3BkJ8OvGv6YMIL1hvXsqtqYIJb0HvK5u1v3PBls6dCjCR89WWhbBoljl6ceBt8N0zUJQJ8ZNg1FO/9lDaS20tLHsANrx7ykKLGVJ+Pczen18Ps2pK5x7mn90afm5t15HjTHprDUFernxy5wV09nBpfL95afDLP2Hjv6DiRD3Kca9C8s1Nbnp1TS1//WI37/yYxoXRAbz2+374uje8z/XpeUx79xfcXZz41/RB9HQtgA9vhCOb4eJHzPuRHei+mC2dOxVgIuezlJXw4U1QXWYODrnib+azZOdKba05JN/iaAaVb7dm9fhO9fP+XG5+Zx1xod58cNuQ02bFNgyDrKIK0nJKOJBbQsaxHIIPLMawOFCVeBNDIv2J6eLV8OMDpygoreSeBZtYlZLD1GERPH5l7Fnv2+06cpyb5q6jpraWeVMHkRTsCksfNO9HRl5i9sYamQ5o77EilmzJJDW7GAeLBScHC44ODjg6gKODw4m/m6/AygxGH/g/jCufJyomoTm/vjq2dO5UgImc745sNat3xE1o1fxnHcGK7UeY+f5GLukVyJi+waTnlpJ+omL/gdxSyqpq6tZ1drTQtbM7FdW1HC4wa036ujszKMKPwZH+DIn0IzbYu16g7csq4rb31nO4oIy/jI9n0sCuTW7bgdwSbpy7ltziSubcPIBhUf5mKarlj4CLu/nQc8wYwCydtXhLJku2ZLL7aBEOFogI8AADqmsNak68qmsNag2DmppqJtWu4EHLB1ThxJ4LX2bgqEkt+h3a0rlTASYiduXfPx/gic+3A+DkYKGbnzsRAR5093enR4CHeU/O34NQX7e6nlNGfilr9+fx8/5c1qblcTCvFABvNycG9TDDzLuTM08v2YmbsyNv3ZRM/+7Nnxfs2PFybp67jrScEl65IYkxfUMgey98Og2ObmNH1xuYXXY9v2SY++/fvTPjEkO5Ij6YIK9GeqZ5abDobjiwGnqOhrEvt+ryry2dOxVgImJ39mcX4+TgUC+kmiOzoIy1abl1oZaeawZKfJgPb93Un9CGBoo0UWFpFVPfXcfmQwXMHheHi6MDX2xO55KD/2Ca03L2O0awLvkFhg0dRlc/98Y3VFsL6+eaZcYcHM1ak0lTWl1/0ZbOnQowEZGzOFpYzt5jRWd/YLuJSiurmfnvjXy/NxuACH93xiWGMrnzLkK/e8gcETrmr+aQ/4YCKT/d7HWlrzKf0Rv3SvOrjjTCls6dqsQhInIWwT5uBPu0bHBJQ9xdnJhz8wC+2HaEyEAP4sN8Tjz4HQO9h8Lnd8LSB8yHyce+cmJ2b8xe14Z34Kv/Z1YzGfuKOXrSBqvetwUFmIiIFbg4OTC+X9jpb3h1gSmfmtPQfP0UHL4QrnnbHKG56G5I+94cuTjuNfBt+iASe6QAExHpaBwczMkxIy6ET6bDu1eBs7vZ07rqJXMGgPO013UqBZiISEcV2g/u+MGsmlJ8zCzr1bm7tVvVYSjAREQ6MldPGPuStVvRIXWc+iUiIiLNoAATERGbpAATERGbpAATERGbpAATERGbpAATERGbpAATERGbpAATERGbZLfV6AMCAoiIiKi3LDs7m8DAQOs0qB3Y2/GA/R2TvR0P2N8x2dvxQOuOKT09nZycnDZuUfuw2wBriC1NE9AU9nY8YH/HZG/HA/Z3TPZ2PGCfx9QQXUIUERGbpAATERGb5Dh79uzZ1m7EudS/f39rN6FN2dvxgP0dk70dD9jfMdnb8YB9HtNvnVf3wERExH7oEqKIiNik8yLAVqxYQUxMDNHR0Tz77LPWbk6biIiIID4+nqSkJAYMGGDt5jTbtGnTCAoKom/fvnXL8vLyGDVqFD179mTUqFHk5+dbsYXN19AxzZ49m7CwMJKSkkhKSuKLL76wYgub59ChQ4wYMYI+ffoQFxfHyy+/DNju99TY8djyd1ReXs6gQYNITEwkLi6OJ598EoC0tDQGDx5MdHQ0119/PZWVlVZuaTsx7Fx1dbURGRlppKamGhUVFUZCQoKxY8cOazer1bp3725kZ2dbuxkt9v333xsbNmww4uLi6pb98Y9/NJ555hnDMAzjmWeeMR5++GFrNa9FGjqmJ5980nj++eet2KqWy8zMNDZs2GAYhmEcP37c6Nmzp7Fjxw6b/Z4aOx5b/o5qa2uNoqIiwzAMo7Ky0hg0aJCxZs0a47rrrjMWLFhgGIZh3HHHHcY//vEPazaz3dh9D2zdunVER0cTGRmJi4sLkydPZtGiRdZu1nlv+PDh+Pn51Vu2aNEibrnlFgBuueUWPv/8c2s0rcUaOiZbFhISQnJyMgBeXl7ExsZy+PBhm/2eGjseW2axWPD09ASgqqqKqqoqLBYL3377LRMnTgRs6ztqLrsPsMOHD9O1a9e6v4eHh9v8P1ow/+GOHj2a/v378/bbb1u7OW3i2LFjhISEABAcHMyxY8es3KK28dprr5GQkMC0adNs5nLbb6Wnp7Np0yYGDx5sF9/TqccDtv0d1dTUkJSURFBQEKNGjSIqKgpfX1+cnJwA+znnNcTuA8xerV69mo0bN7J8+XJef/11fvjhB2s3qU1ZLBYsFou1m9FqM2fOJDU1lc2bNxMSEsJDDz1k7SY1W3FxMddeey0vvfQS3t7e9d6zxe/pt8dj69+Ro6MjmzdvJiMjg3Xr1rF7925rN+mcsfsACwsL49ChQ3V/z8jIICwszIotahsnjyEoKIgJEyawbt06K7eo9bp06cKRI0cAOHLkCEFBQVZuUet16dIFR0dHHBwcuP32223ue6qqquLaa69lypQpXHPNNYBtf0+NHY8tf0cn+fr6MmLECNasWUNBQQHV1dWA/ZzzGmL3ATZw4EBSUlJIS0ujsrKShQsXMm7cOGs3q1VKSkooKiqq+/NXX31Vb+SbrRo3bhzvvfceAO+99x5XX321lVvUeidP9ACfffaZTX1PhmEwffp0YmNjefDBB+uW2+r31Njx2PJ3lJ2dTUFBAQBlZWWsXLmS2NhYRowYwSeffALY1nfUbNYeRXIuLFu2zOjZs6cRGRlp/PnPf7Z2c1otNTXVSEhIMBISEow+ffrY5DFNnjzZCA4ONpycnIywsDDjn//8p5GTk2NceumlRnR0tDFy5EgjNzfX2s1sloaO6cYbbzT69u1rxMfHG2PHjjUyMzOt3cwmW7VqlQEY8fHxRmJiopGYmGgsW7bMZr+nxo7Hlr+jLVu2GElJSUZ8fLwRFxdnPPXUU4ZhmOeIgQMHGlFRUcbEiRON8vJyK7e0fagSh4iI2CS7v4QoIiL2SQEmIiI2SQEmIiI2SQEmIiI2SQEmIiI2SQEm0kyOjo51lcuTkpLadIaD9PR0m3oOScSanKzdABFb06lTJzZv3mztZoic99QDE2kjERERPPzww8THxzNo0CD27dsHmL2qSy+9lISEBEaOHMnBgwcBs3jxhAkTSExMJDExkZ9++gkwi7PefvvtxMXFMXr0aMrKygB45ZVX6NOnDwkJCUyePNk6BynSgSjARJqprKys3iXEDz/8sO49Hx8ftm3bxt133839998PwD333MMtt9zC1q1bmTJlCvfeey8A9957LxdffDFbtmxh48aNxMXFAZCSksKsWbPYsWMHvr6+fPrppwA8++yzbNq0ia1bt/Lmm2+e46MW6XhUiUOkmTw9PSkuLj5teUREBN9++y2RkZFUVVURHBxMbm4uAQEBHDlyBGdnZ6qqqggJCSEnJ4fAwEAyMjJwdXWt20Z6ejqjRo0iJSUFgOeee46qqiqeeOIJxowZg6enJ+PHj2f8+PF180CJnK/UAxNpQ6dOLdLSaUZODTRHR8e6quLLli1j1qxZbNy4kYEDB9YtFzlfKcBE2tDJy4kffvghQ4cOBeCCCy5g4cKFALz//vtcdNFFAIwcOZI33ngDMO97FRYWNrrd2tpaDh06xIgRI3juuecoLCxssBcocj7RKESRZjp5D+ykMWPG1A2lz8/PJyEhAVdXVxYsWADAq6++ytSpU3n++ecJDAxk3rx5ALz88svMmDGDuXPn4ujoyBtvvFE30/Fv1dTUcOONN1JYWIhhGNx77734+vq285GKdGy6BybSRiIiIli/fj0BAQHWborIeUGXEEVExCapByYiIjZJPTAREbFJCjAREbFJCjAREbFJCjAREbFJ/x+jH84TZrA6hwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "image/png": {
       "height": 300
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Image\n",
    "Image(save_dir+'/loss_history_class.png', height=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 317
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1245,
     "status": "ok",
     "timestamp": 1590649688894,
     "user": {
      "displayName": "Zigzag13 Mak13",
      "photoUrl": "",
      "userId": "17521176493425296049"
     },
     "user_tz": -180
    },
    "id": "Owtk8IQiCKlh",
    "outputId": "a18c2706-7a24-4f50-b759-4bb4015001c0"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbAAAAEgCAYAAADVKCZpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de1xUdf7H8ddwvwrIXVARVOQioo5aWSm5mrWlqWS5VmYl+3MrW3+55Xb5ZW0X3dpdza64Zm7bT3NNs19qF1Gzi2ZoeAdvoHG/yB2GYZjz++PoGAkKyjBn8PN8PHgIM+ec+Rym5s33e77n+9UpiqIghBBC2BkHWxcghBBCXA4JMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHZJAkwIIYRdkgATQghhlyTAhBBC2CUJMCGEEHbJydYFWEtAQAARERG2LkMIIexKTk4OpaWlti6jTbpsgEVERJCenm7rMoQQwq7o9Xpbl9Bm0oUohBDCLkmACSGEsEsSYEIIIeySBJgQQgi7JAEmhBDCLkmACSGEsEtddhi9EEJoWU2Dib2nymkwmdu0vbebE/2Dvenu6WLlyuyHBJgQQnQCRVE4UVLL9qxitmUVszv7DI1NSruPE+jtyoAQb/oHexMd4k10sDf9gr3wcLn6Ps6vvjMWQohOYmhsYufJMrZnFrMtq4TTZ+oA6BfkxcyRfbixXyC+Hs5tOlZZrZGjhdVkFlZztKiaf+86ZWm96XTQq7sH0WdDbcKgHvQL9rbaeWmFBJgQQlxCvbGJg/mVNDReurtPQSG7tJZtmcV8f6KMBpMZN2cHRkYFMOvGSEb3D6Rnd4/LqmNU/0DL901mhVNltRwtOh9qmYXVbDlSxKBwXwkwIYS4GjU2mdmfW8F3x8v4/kQpe09VYGxq27Wqc3p192Da8F4kDQhiRJ/uuDk7dmiNjg46IgO9iAz0Ynx8qOVxQ2MTOl2HvpRmSYAJIa56ZrNCZmE1358o5fsTZfxwsoxaoxoEsaHdmHFdb66J9Kebe9u6+wK8XInw90BngyTp6KDUMs0F2JIlS1i2bBmKojBr1iz++Mc/smDBApYtW0ZgoNp8fvnll7n11lttXKkQojMYGps4XFDFwbxKcsvrMZrMNJiaaDCZz35vtjx27ufGJjNODg64ODng6nT+X1cnx2aPuTg6UFBlYOeJMs7UGgHoE+DJHYPDGNk3gGsj/fGTUX+apakAO3jwIMuWLWP37t24uLgwfvx4brvtNgDmzp3LvHnzbFyhEMKaGkxNZBVWsz+3kgO5lezPq+RoUTVNZnW0nquTA27OF4aQq7Mjro4OeLo64efhgLOjAyazYgm1aoOJsnMh12SmodGMsUkNvm5uzozuH8h1fQO4LsqfHr7uNv4tiLbSVIAdOXKEESNG4OGhXuAcNWoU69ats3FVQlydGpvMHC2qtgTJgdxKsoqq8fNwJsLfk8hATyL8PYkI8KRPgCe9untcsvtKURTqjE2U1jRQWmOkrKaBouoGDudXcSCvgqzCasvQcj8PZwaG+zJmQBADw31ICPchpJubTbrlhDZpKsDi4+N5+umnKSsrw93dnU2bNqHX6/H39+eNN97gX//6F3q9nr/97W/4+fnZulwhugxTk5njJTWWls+BvEoOF1RhPDtM29vNiYRwH+69pjfldUZySmv54lCRpdsN1KHcPXzciQjwIMLfk27uzpTVNFBWY6S01khpdQNltQ0YWhjJ183NiYHhPjx4fSQJ4T4MDPMh3M9dwkpclE5RlPbfSWdFy5cv56233sLT05O4uDhcXV3585//TEBAADqdjmeffZaCggLee++9C/ZNTU0lNTUVgJKSEk6dOtXZ5QuhCbUNJg7mVXKm1ki1wUSVoZFqg+nsVyM1Dee/rzaYyK+stwSLl6sTcT26qUES7ktCmA+9WxmQUFnfSE5pLTlltWSX1pJTWkt2WR05pbXUNpjw93LB39MVfy8XAr3Uf/29XPH3dCHA25UAT1cCvF2kZaUher3ebhYD1lyA/dJTTz1FeHg4f/jDHyyP5eTkcNttt3Hw4MGL7mtPb4IQV6qw0kD6qTOk55Sz51Q5hwuqLNeNfsnTxRFvN2e83ZzwdnPC6+z3wd5uZwPLhz7+njg4XHmYKIoioWSH7OmzU1NdiADFxcUEBQVx+vRp1q1bx65duygoKCA0VL3PYf369cTHx9u4SiE6jqIoFFU1YDKbm4+Sc3RoMUjMZoWjxdWk55STnnOG9FPl5JbXA+Dm7MDgnn78YXQUQ3r7EdLN7WxYOePl6oRjBwRTW0l4daCf/g0Vp2H0n7lqbvJqA80F2JQpUygrK8PZ2Zk333wTX19fHn30UTIyMtDpdERERPDuu+/aukwhLpupyczhgio1gM62moqrG1rc1tlRd8HQ73PdgqDOi6fv7cfMkX3Q9/Yjtkc3nB1lkYku5eR2+PRRUMzgGQjDZ9m6Is3QXIB98803Fzz2wQcf2KASITpGtaGRvacr2HO2tfTT6QrqG5sACPN157oofwb38sPd2ZGGpgvvaTL+6j4nD1cnhvbyQx/hR6/utrlZVnSSqnz4+CHw7we+PeHzP0OPIRA+1NaVaYLmAkwIe6MoClUGE/kV9RRU1pNXYVC/r6gnq6iGrMIqzAo46CC2RzfuGtaTob3VAAr1kXuORCuaGmHtA2Csg/s3qq2vd0fBf2bA73eAR3dbV2hzEmBCtJGiKBzKr+Lb46WcKqsjv6L+bGgZqGkwNdvWyUFHiI8bfQI8GXdTP4ZFdCexly9ervK/nGijLQvg9E6YshwCo9XHpq6E926GdSnwuzXgcHV3F8v/TcL+1J2B6kIIirH6Be0qQyPfHStlW1Yx27NKLNeq/D1d6OHrTp8AT0b2DSDM150evu6E+roR5utOgJdrpw6YsGtFh6E6v23b+vQ8/2HelR3+FHa+AcNmwcDk84+HDYHxr8DGx+Gbv8GoP9muRg2QABP2w1AJO99Uv4w14NcH4qeo/4MHxXTISyiKwvHiGrZlFbM1s5j0nHJMZgVvNydu7BdI0oAgRvUPJNDbtUNez+aaGiFvL5Rnt217Z3eIvhUc2zap7SVl74CVt7dvn4gbYMR/QfQt4NAFJ64tOwEbHoawoXDzSxc+r38QTu+CbS9BuB6ikjq/Ro3Q9H1gV8Ke7mUQl2Csg92p8N1iqC+H2InQZxQc+VT9AFTMEBQHA6eogeYX0a7DV9Y38sPJMnYcK2FbZgl5FeqQ9AEh3oyODiIpOpAhvf26xug+sxmKD8HJryH7azj1vfrHQHtc8we1FdARtSxLgroytZvsUq1pRYGfd8EPqVCVC769YcTvYfA94OZz5fVoQWM9/HOsen6/3wG+vVrerqEG/jkGakvhv76Bbj06rAR7+uyUABPaZTLC3pWw41WoKYK+Y+GmZ6BH4vltqovg8CdwYC3k7lYfCx8G8ckQNwm8gy84rKGxifSccr47Ucr3x0s5kFeJWQF3Z0dG9g0gaUAgSdFBlzepq6JAbQkUHYLiI9DUoLYYQhPB0QYdHooCZ06qYZW9Q/2qK1Of8++r/iEQOQqC49vWHbvjNdj/EczeCYH9r6y2A2vh4wdhUioMuqvt+zWZIPMz+OEd9RqRixckTlfDzD/qymqytQ0Pq/d8TV8L/cZefNuSLEhNgpCBcP9nHdYqtqfPTgkwoT1NJvVDcvtCqDwNva6DMf8Dva+9+H7lp+Dgx+pX0UHQOUDE9ZgDYylo8iazypW9Zc78UOxIYZM3FTofonsGMzLKn+v6BjC4ly+uTu3okjJUQUnm+bAqPqx+nQuIX3LtBr1HqmHRZ5R1rt8pCtQUqy2s4iNQeAByvoXKn9XnvXucf/0+N4JPWPtfo6YElg6BXtfA9P9cfq2mBnhDr7acUnZc/mCE/J9g1zvqe242Qb9xcM1/QWSS/d3wu/cD+PQRuPFP6h9qbXHuj4BrH2m5u/Ey2NNnpwSY0A6zGY5sgK0vQdkxtdUy5lmIGmP5MDI1qUtj1DQ0n9+vpuH891WGRtwrjjGg9EviK3fg11iEp87Q8ms6e4JXoDpE2bVb2z70zCYoO6mG6zkuXmooBcWo3ZlBMRAUqx4ve4faAjr59flrTZ5Baoj0uVENlXZ2e2Koah6axUfUIK0/c34bzyA1aCJHQZ/RauukIz7Uv18KXz7TtlZCq8d4A758Gu79pGOu4VQXQvp76ldtCQTGwPiXIeqmKz92ZyjYD8vHQs8RcO/69l3b2zgPflwGUz+A2AlXXIo9fXZKgAnbUxSqD26i4csXCKjOpMg1gg3d7+cbx2upbmiyTDhbbTBZbgC+GBcnB7xd1bn+urk7Ex/mww29Pbk2RMFXqVCvG9QUqx90taVn/y2Ghuo2FqxTAyc4Vg2poFh1dFxbWhEVp9VAO3cNqqZIfdynF3i0cYWFujPnW1Xwi/A8W8u5ujwD2ng+7WQywlvXqB+ys79vf9dVfTksSVQHKdzbwcslNRrg0Dp1hF7ZcXWwx28WqINPtMpQqd7fZTLA779R/6BqD1MDrLgFSo9ByvYr7ka1p89OCTBhU8X7t2D44nl61e7nlDmIt7iTr11G4enuapl0ttsvJp89N6ffue+7nXvM8rxT+7oBbUlRoPSoGmanv1cv4LeFq3fzlp5vr87vLsvaDKvuhvEL4ZrZ7dv3y2fVVtx/fQshVprX1Fin3ke1+10I6A+TU6HHYOu81pVQFPjoHvX3OXOT2mK+HBWn4d0boVsYPLTligLbnj47JcCuBsY6qC5o3tqwfF/SvEXi4Nj8Wk33SKt8OJ7M2EHDVy8QU/sjhYof3/V4gIG3P0z/Hv4d/lrCChQF/j0Z8vbAoz+BZxvft4rTsFSv3vpwx1vWrRHgeJo6MKK2BEbPh5FzbTOYpjXnulLHvQTXPXJlxzr2FXyYDIn3wB1vXvZh7OmzU0PvpOhwJqN6M+SOV6Gx7sLn3XzU6ySegerNoRHXg7FW7eI6/Im6jU/Ps9dqzl747xZ62eUoikLG3p00bXkRff13lCvefNVrDvF3zGWKv0yLY1d0Orj5FXj7OvV+pNv+3rb9tr6k7pv0lHXrO6fvGLWbc+N/w9YX4eiXMPld9Q8zWzLWwtd/VVuiMbfDtQ9f+TH7jVUHgOx4Vf0DNGHqlR9T4yTAuqqTX8OmeWoX1YDb1C/PwPMDFjwCwMml5X0VRb2ZMnu7epysTZDxofpcQPT5gQfdo84eq/tFLzo3mRW+/WE35u2vMMqwnTqdOz9E/BcDJj7JWD8JLrsVNACGPQg//lP9Nzju4tsX7FNHl17/R/AJ75waQf3vM3kFRP9WncHi7evVAR5DZnR+16uiQOZG2Pykeq9X4j1wy8KOq2P0n8HBWR2NeRWQLsSuprpQHSF24D/qjZ63vgr9b76yY5rNUHSg+c2vzVp0OvDwB68gFI8ADK7dKdf5UNzkTV6jN025e7mlcQtNOkdO9JlO1B1P4+bTzgvVQpvqzsDrgyE0Ae77tPUPYkWBf01Uh/Y/lmG7G48rc+GT2WovQ//xMGEpeAV1zmufyYbNT8CxL9X77n77t8u/5mVF9vTZKS2wrqLJpP4lvO0ldTTTqCfh+rkdM/rKwQFCB6lfI+fQaDRQdTKdMwXZVJXmY6gopKm6BMfyUtyKivBTjhKgq6SHrp5EoBEn8qLuoufE/yHO5/K7IIUGeXSHpKdh85/UlkXMbS1vdyJN/eNn/CLbzprhEw73blBvgt6yAN66Fm56GsL0aje6kxWmCDM1wHdL1JGRDk5q1+vwFG1di7NT0gLrCn7+ETbOVf+6jboJbn2t3UNpFUWhpKaBU2V1lNU0UFpjpKzGSFltA6WWnxsoqzVSUdfYbF9nRx09u3vQx9+TiABP+pz9ivB1JNSxBgcXj7Zf5Bf2p8kE74xUP6gf/uHCEDA3qSPkjDXw8I+td113tuIj6qzuhfvVn3WO6v83v74dwS/i8udcPJ4Gm/4EZ05A3GT1ZuMOnPbJGuzps1NzfwIsWbKEZcuWoSgKs2bN4o9//CNnzpzhrrvuIicnh4iICNasWYOfXxvvmenK6s7Aludg77/UWRbuXKnOE3iJ/vRqQyNHi6rJKlTXqsoqqiarsJryXwUTgI+7MwFeLvh7uRId4k2Alyv+nq74e7lYQquHrxtOrc4TKO9Tl+foBDe/rI5K3PW2eo3rl/Z/pM6MkrxCO+EF6i0IKdvV+6fOzV5SfES9Vnd4A3D2b3snd7V1FhSjho9nkHqPnefZ68leQeDu1zzkqvLhi6fg0Hp1yq6OumFbNKOpFtjBgwe5++672b17Ny4uLowfP5533nmH1NRUunfvzvz581m4cCHl5eUsWrTooseyp78i2k1RYP8a+Hy+ehPkNbPVIcKu3hdsamoys+VIEftyK8kqVIPq3GS1AB4ujvQP9mZAiDf9g72JDPQkwMuVQG9X/DxccHHqAhPYis7xv3dBznfw6J7zc1A21sPSoeAVDLO22s/0TsZadZqw4iPqci/Fh9W5B2uKQGnhZnqdg3od2DNI7W3I26vO2HLjPLhujnW6Jq3Enj47NdUCO3LkCCNGjMDDwwOAUaNGsW7dOjZs2MD27dsBmDFjBqNHj75kgHVZdWfgs7nqMPfw4XD74hZHfzU2mVm/N483th3n9Jk6nBx0RAV6MbS3H78b0YvoYG+iQ7wJ83XHQdatEh1h3EvqDB1bX4CJZ+9D2vU2VOXB5GX2E14ALp7qTCFhQ5s/bjaDoeIXM7n8+uvsPZX9xqozgLR3ijDRLpoKsPj4eJ5++mnKyspwd3dn06ZN6PV6ioqKCA1VL/6HhIRQVFRk40pt5PgW+ORhdbLYMc/ByMcu6Js3msys3ZPLW9uPk1tez8AwH1LvHcro6CBpTQnrCuirzgi/8011IUafnvDtP9T1wyJG2rq6juHgoA5c8egODLB1NVc9TQVYTEwMTz75JOPGjcPT05PExEQcHZt/QOt0OnSt/CWXmppKamoqACUlJVavt9MY6+Cr/1En7AwcANPXqCMCf6HB1MSa9Fze3nac/EoDg3r68sLEOJKig1r9fQnR4UY9AftWq93boYPUgRu/WWDrqkQXpalrYL/21FNPER4ezpIlS9i+fTuhoaEUFBQwevRosrKyLrqvPfXjXlTeHnWkVNlxdcmEm54FZzfL04bGJlbvPs07X5+ksMrAkF6+PPab/tzYL0CCS9hG+gr47OxAjqH3w+1LbFqOaB97+uzUVAsMoLi4mKCgIE6fPs26devYtWsX2dnZrFy5kvnz57Ny5UomTpxo6zKtr8mk3jfy9SLwDlFvEo0cZXm6ytDImh9/5t0dJympbmB4RHf+NnUQ10X5S3AJ2xpyH/y4XB06PvrPtq5GdGGaC7ApU6ZQVlaGs7Mzb775Jr6+vsyfP5+pU6eyfPlyevfuzZo1a2xdpnWVnVBbXXnpMHAq3PoqipsPx4qq2ZZZzLasYtJzyjGZFa6L8mfptMFcEyn3WQmNcHBU17SqK1X/+BLCSjTdhXgl7KkZ3Mye9+HzP4OjCw3jX+Nb1xvZllXMtswSy/D3ASHeJA0I4ua4EBJ7+tq2XiFEl2JPn52aa4Fd1Q6shf97jLzu1/BXt8fYvFaH0ZSOh4sj1/cN4JGb+jI6OpBQHw0vzieEEJ1EAkwjjIWHYf0j7DNHMy3/D/QK9ODea4JIig5iWB8/+1mkUQghOokEmAYczM7D+4OpeDQ5s3nAK6TdfA29/T1tXZYQQmiaBJgNNZiaeH3LUfp/9zi3OeZxIGkF/zN6jK3LEkIIuyABZiP7cyv403/2M6x0HROdv8Nww1Mkjr7D1mUJIYTdkADrZA2mJl5PO8Y7X5/kBo/TvOD6b4gah1vSn2xdmhBC2BUJsE50rtWVVVTNfYO8WVCwFAddKEx6V51jTQghRJtJgHWCX7a6ArxcWDFjKEl7H1WXZnjwi7MTgwohhGgP+bO/E7yz/SRvbjvBpMFhfDl3FEklH8CxL2H8Kxcu1yCEEKJNpAXWCfacLic2tBuv3TkITm6HbS/DwDth2EO2Lk0IIeyWtMA6QVZhFQNCvNVlxj9+CPz7wW2L7WuBPyGE0BhpgVlZRZ2RoqoGYoLc4D8z1bW97t8Irl62Lk0IIeyaBJiVZRVWAzC+KBV+3gVTlkNgtI2rEkII+yddiFaWVVTNzQ4/0jNzubrM+sBkW5ckhBBdggSYlZWcOsxrLu+g9BgCN79k63KEEKLLkACzpsZ6phx/CnRO6KauBCdXW1ckhBBdhuYC7B//+AdxcXHEx8czbdo0DAYD999/P3369CExMZHExEQyMjJsXWabKBvnEWHKZn2f58C3l63LEUKILkVTgzjy8vJ4/fXXOXz4MO7u7kydOpXVq1cD8Oqrr5KcbEfXj/Z+gC7j37xuuoPu/cfZuhohhOhyNNcCM5lM1NfXYzKZqKuro0ePHrYuqf0K9sOmeZwJvpbFpmT1HjAhhBAdSlMBFhYWxrx58+jVqxehoaH4+Pgwbpzaenn66adJSEhg7ty5NDQ02LjSizBUwpr7wN2PDVEvYMaB/hJgQgjR4TQVYOXl5WzYsIHs7Gzy8/Opra3l3//+N6+88gqZmZn8+OOPnDlzhkWLFrW4f2pqKnq9Hr1eT0lJSSdXDygKfPIHqDgNd75Pxhlnevi40c3NufNrEUKILk5TAbZlyxb69OlDYGAgzs7OTJ48me+//57Q0FB0Oh2urq7MnDmT3bt3t7h/SkoK6enppKenExgY2MnVAzvfgMzPYOwL0OsasgqriZbWlxBCWIWmAqxXr17s2rWLuro6FEUhLS2NmJgYCgoKAFAUhU8++YT4+HgbV9qCUzvhq+cg5na49mEam8ycKKkhOqSbrSsTQoguSVOjEEeMGEFycjJDhgzBycmJwYMHk5KSwi233EJJSQmKopCYmMg777xj61KbqymG/9wPfr1h4pug03GypIbGJkUGcAghhJVoKsAAnn/+eZ5//vlmj23dutVG1bSBuQnWPgCGCrhnLbj5AOoUUoB0IQohhJVoLsDszraXIecbmPgWhAy0PJxVWIWTg46oQJl1XgghrEFT18DsztEv4JvXYPC9MHh6s6eyCquJDPTExUl+xUIIYQ3y6Xq5jHWw/vdqq+vWVy94OrOwmv7B0n0ohBDWIgF2uU5shfpyGPsXcHZv9lRNg4nc8noZwCGEEFYkAXa5MjeqAzYirr/gqXOLWMoQeiGEsB4JsMvRZIKjn0O/m8Hxwlk2jp4dgSgtMCGEsB4JsMvx8y6oPwMDftvi01mF1Xi6OBLm697i80IIIa6cBNjlyNwEji7Qd0zLTxdW0T/EGwcHXScXJoQQVw8JsPZSFHW+w8jR4HphF6GiKOociDICUQghrEoCrL2KD0PFqVa7D0uqGyiva5QZOIQQwsokwNorcyOgg/63tPx0oUwhJYQQnUECrL0yP4PwYeAd3OLT50cgyhB6IYSwJgmw9qjMhYJ9MODWVjfJLKwm0NuV7p4unViYEEJcfSTA2iNzk/rvgNta3SSrsFru/xJCiE5glQBbunQp5eXl1ji0bWVthID+ENCvxaebzApHi2QORCGE6AxWCbCioiKGDRvG1KlT+fzzz1EUxRov07nqKyDnW4huvfvwVFktDSazDOAQQohOYJUAe/HFFzl27BgPPvgg77//Pv369eOpp57ixIkTl9z3H//4B3FxccTHxzNt2jQMBgPZ2dmMGDGCvn37ctddd2E0Gq1R9sUd+wrMpkt2H4JMISWEEJ3BatfAdDodISEhhISE4OTkRHl5OcnJyTzxxBOt7pOXl8frr79Oeno6Bw8epKmpidWrV/Pkk08yd+5cjh8/jp+fH8uXL7dW2a3L/Ay8giFsaKubZBVVo9NBvyAJMCGEsDarBNiSJUsYOnQoTzzxBCNHjuTAgQO8/fbb7Nmzh48//vii+5pMJurr6zGZTNTV1REaGsrWrVtJTk4GYMaMGXzyySfWKPsiRTXA8S0QfQs4tP4ryyqsJsLfE3cXx04sTgghrk5O1jjomTNnWLduHb179272uIODA5999lmr+4WFhTFv3jx69eqFu7s748aNY+jQofj6+uLkpJYaHh5OXl6eNcpuXfYOMNZctPsQkCmkhBCiE1mlBXbLLbfQvXt3y89VVVX88MMPAMTExLS6X3l5ORs2bCA7O5v8/Hxqa2v5/PPP2/y6qamp6PV69Ho9JSUll38Cv5b5Gbh4QZ8bW93E0NhETlkt/eX6lxBCdAqrBNjs2bPx8vKy/Ozl5cXs2bMvud+WLVvo06cPgYGBODs7M3nyZL777jsqKiowmUwA5ObmEhYW1uL+KSkppKenk56eTmBgYMecjNkMWZuh72/AybXVzY4V1WBWZACHEEJ0FqsEmKIo6HTnlxJxcHCwBNDF9OrVi127dlFXV4eiKKSlpREbG0tSUhJr164FYOXKlUycONEaZbcsbw/UFLU6ee85mYVVgMyBKIQQncUqARYZGcnrr79OY2MjjY2NLFmyhMjIyEvuN2LECJKTkxkyZAgDBw7EbDaTkpLCokWL+Pvf/07fvn0pKyvjwQcftEbZLcvaCA5O0G/sRTc7WlSNq5MDEf6enVSYEEJc3XSKFe4yLi4uZs6cOWzduhWdTseYMWNYvHgxQUFBHf1SrdLr9aSnp1/5gd4YBt16wH0bLrrZvct/oLzOyGeP3nDlrymEEDbSYZ+dncAqoxCDgoJYvXq1NQ7duUqPQelRGDbrkptmFVZzQ78Ouu4mhBDikqwSYAaDgeXLl3Po0CEMBoPl8ffee88aL2c9mRvVfy8y+zxAea2R4uoGokO8LrqdEEKIjmOVa2D33nsvhYWFfPHFF4waNYrc3Fy8ve1wcEPmRghNBJ/wi29mWcRS1kYwA1MAAByPSURBVAATQojOYpUAO378OH/5y1/w9PRkxowZbNy40XIfmN2oLoLcHy85+hAg6+wIRBlCL4QQnccqAebs7AyAr68vBw8epLKykuLiYmu8lPUc3QwobQuwohp8PZwJ8m79PjEhhBAdyyrXwFJSUigvL+fFF19kwoQJ1NTU8Je//MUaL2U9mZvAtzcExV5y06zCKqKDvZvd+yaEEMK6OjzAzGYz3bp1w8/PjxtvvJGTJ0929EtYX0M1nNwOwx6CS4SSoigcLaphypCWZwcRQghhHR3ehejg4MBf//rXjj5s5zqeBk0Nbeo+zC2vp6bBJHMgCiFEJ7PKNbDf/OY3vPbaa/z888+cOXPG8mU3sjaBe3foOeLSm8oilkIIYRNWuQb20UcfAfDmm29aHtPpdPbRndjUCEc/V5dOcbz0ryerSA2w/rKMihBCdCqrBFh2drY1Dts5Tn0PhkqIvvjNy+dkFVYT5uuOt5uzlQsTQgjxS1YJsH/9618tPn7fffdZ4+U6VuZGcHKHqJvatHlWYbV0HwohhA1YJcB+/PFHy/cGg4G0tDSGDBliHwE2er46dZSLxyU3NZrMnCipYUxM501SLIQQQmWVAFu6dGmznysqKrj77rut8VIdz6M7RI5u06YnS2swmRVZA0wIIWzAKqMQf83T09O+r4u1IssyB6IEmBBCdDartMBuv/12y6wUZrOZw4cPM3XqVGu8lE1lFlbj5KAjMkBmoRdCiM5mlQCbN2/e+RdwcqJ3796Eh198RneArKws7rrrLsvPJ0+e5IUXXqCiooJly5YRGKiut/Xyyy9z661tGyVoTVmF1UQFeuHi1CkNWSGEEL9glQDr1asXoaGhuLm5AVBfX09OTg4REREX3S86OpqMjAwAmpqaCAsLY9KkSaxYsYK5c+c2C0YtyCqsZmhvP1uXIYQQVyWrNB3uvPNOHBzOH9rR0ZE777yzXcdIS0sjKiqK3r17d3R5HaLOaCKvop7+wdJ9KIQQtmCVADOZTLi4uFh+dnFxwWg0tusYq1evZtq0aZaf33jjDRISEnjggQcoLy/vsFovV35FPQA9u196uL0QQoiOZ5UACwwM5NNPP7X8vGHDBgICAtq8v9Fo5NNPP7W02mbPns2JEyfIyMggNDSUxx9/vMX9UlNT0ev16PV6SkpKruwkLiG/wgBAD193q76OEEKIllnlGtg777zD9OnTeeSRRwAIDw9vdXaOlmzevJkhQ4YQHBwMYPkXYNasWdx2220t7peSkkJKSgoAer3+cstvk3MtsFAfN6u+jhBCiJZZJcCioqLYtWsXNTU1AHh5te860apVq5p1HxYUFBAaGgrA+vXriY+P77hiL1N+RT0OOgjuJgEmhBC2YJUuxKeeeoqKigq8vLzw8vKivLycZ555pk371tbW8tVXXzF58mTLY0888QQDBw4kISGBbdu28Y9//MMaZbdLfqWB4G5uODvKEHohhLAFq3z6bt68GV9fX8vPfn5+bNq0qU37enp6UlZWho+Pj+WxDz74gAMHDrB//34+/fRTS2vMlvIr6qX7UAghbMgqAdbU1ERDQ4Pl5/r6+mY/dwX5FfUygEMIIWzIKtfApk+fzpgxY5g5cyaKovD+++8zY8YMa7yUTSiKQn6lgZvjQmxdihBCXLWsEmBPPvkkgwYNYsuWLeh0Om6++WZOnTpljZeyibJaI0aTWboQhRDChqw2AiE4OBidTsd//vMftm7dSkxMjLVeqtOdG0IvXYhCCGE7HdoCO3r0KKtWrWLVqlUEBARw1113oSgK27Zt68iXsTm5iVkIIWyvQwNswIAB3HDDDXz22Wf07dsXQBND3juatMCEEML2OrQLcd26dYSGhpKUlMSsWbNIS0tDUZSOfAlNyK+ox83ZAT8PZ1uXIoQQV60ODbA77riD1atXk5mZSVJSEosXL6a4uJjZs2fz5ZdfduRL2VRBpYEevu6WRTuFEEJ0PqsM4vD09OR3v/sd//d//0dubi6DBw9m0aJF1ngpm8irqKeHj3QfCiGELVl9HiQ/Pz9SUlJIS0uz9kt1GvUmZhlCL4QQtiQT+bWT0WSmpKZBBnAIIYSNSYC1U1GVAUVBuhCFEMLGJMDaKU+G0AshhCZIgLVTQeW5AJNrYEIIYUsSYO10bhaOUOlCFEIIm5IAa6e8inq6e7rg7uJo61KEEOKqpqkAy8rKIjEx0fLVrVs3Fi9ezJkzZxg7diz9+vVj7NixlJeX26zGAlnIUgghNEFTARYdHU1GRgYZGRns2bMHDw8PJk2axMKFCxkzZgzHjh1jzJgxLFy40GY15lcYZACHEEJogKYC7JfS0tKIioqid+/ebNiwwbIg5owZM/jkk09sVld+ZT1hEmBCCGFzmg2w1atXM23aNACKiooIDQ0FICQkhKKiIpvUVGVopNpgki5EIYTQAE0GmNFo5NNPP+XOO++84DmdTtfqJLqpqano9Xr0ej0lJSUdXleBrAMmhBCaockA27x5M0OGDCE4OBhQV3cuKCgAoKCggKCgoBb3S0lJIT09nfT0dAIDAzu8rvxKuYlZCCG0QpMBtmrVKkv3IcCECRNYuXIlACtXrmTixIk2qev8QpbShSiEELamuQCrra3lq6++YvLkyZbH5s+fz1dffUW/fv3YsmUL8+fPt0lt+RX1ODroCPKWABNCCFtzsnUBv+bp6UlZWVmzx/z9/TWxHEtBhYGQbm44OshClkIIYWuaa4FpWZ6sAyaEEJohAdYO+ZX1MoBDCCE0QgKsjcxmhcJKmYVDCCG0QgKsjUprGmhsUughNzELIYQmSIC1kSxkKYQQ2iIB1kYFlTILhxBCaIkEWBtZbmKWhSyFEEITJMDaKK+iHk8XR7q5a+7WOSGEuCpJgLVRwdl1wFqbSFgIIUTnkgBro/zKekLl+pcQQmiGBFgb5VfUEyazcAghhGZIgLWBobGJ0hqjDOAQQggNkQBrg8KzQ+ilC1EIIbRDAqwNZB0wIYTQHgmwNsg/2wILkxaYEEJohuYCrKKiguTkZAYMGEBMTAw7d+5kwYIFhIWFkZiYSGJiIps2berUms61wEJkHkQhhNAMzd2V+9hjjzF+/HjWrl2L0Wikrq6OL774grlz5zJv3jyb1FRQWU+AlyuuTo42eX0hhBAX0lSAVVZWsmPHDt5//30AXFxccHFxsW1RQF6FQYbQCyGExmiqCzE7O5vAwEBmzpzJ4MGDeeihh6itrQXgjTfeICEhgQceeIDy8vJOrSu/op5QGUIvhBCaoqkAM5lM7N27l9mzZ/PTTz/h6enJwoULmT17NidOnCAjI4PQ0FAef/zxFvdPTU1Fr9ej1+spKSnpkJoURaGgQlZiFkIIrdFUgIWHhxMeHs6IESMASE5OZu/evQQHB+Po6IiDgwOzZs1i9+7dLe6fkpJCeno66enpBAYGdkhNVfUmao1NMoReCCE0RlMBFhISQs+ePcnKygIgLS2N2NhYCgoKLNusX7+e+Pj4TqtJFrIUQght0tQgDoClS5cyffp0jEYjkZGRrFixgjlz5pCRkYFOpyMiIoJ333230+opqJQAE0IILdJcgCUmJpKent7ssQ8++MBG1fxyIUvpQhRCCC3RVBeiFuVVGHB21BHg5WrrUoQQQvyCBNglFFSqQ+gdHGQhSyGE0BIJsEtQ7wGT7kMhhNAaCbBLyK8wyCS+QgihQRJgF9FkViisMsgIRCGE0CAJsIsorjbQZFYIlZuYhRBCcyTALiJfbmIWQgjN0tx9YFqSXyELWQpha42NjeTm5mIwGGxdSpfi5uZGeHg4zs7Oti7lskmAXcS5FpiMQhTCdnJzc/H29iYiIgKdTm5n6QiKolBWVkZubi59+vSxdTmXTboQLyK/oh5vNye83ez3LxQh7J3BYMDf31/CqwPpdDr8/f3tvlUrAXYR+ZUyhF4ILZDw6nhd4XcqAXYRchOzEKKsrIzExEQSExMJCQkhLCzM8rPRaGzTMWbOnGlZZaM1b775Jh9++GFHlHzVkGtgF5FfUU9iT19blyGEsCF/f38yMjIAWLBgAV5eXsybN6/ZNoqioCgKDg4ttwlWrFhxydd5+OGHr7zYq4y0wFpRb2yivK5RhtALIVp0/PhxYmNjmT59OnFxcRQUFJCSkoJerycuLo4XXnjBsu31119PRkYGJpMJX19f5s+fz6BBg7j22mspLi4G4JlnnmHx4sWW7efPn8/w4cOJjo7m+++/B6C2tpYpU6YQGxtLcnIyer3eEq5XIwmwVuRb1gGTLkQhRMsyMzOZO3cuhw8fJiwsjIULF5Kens6+ffv46quvOHz48AX7VFZWMmrUKPbt28e1117Le++91+KxFUVh9+7dvPrqq5YwXLp0KSEhIRw+fJhnn32Wn376yarnp3XShdiK8+uASQtMCK14/v8OcTi/qkOPGdujG8/dHndZ+0ZFRaHX6y0/r1q1iuXLl2MymcjPz+fw4cPExsY228fd3Z1bbrkFgKFDh/LNN9+0eOzJkydbtsnJyQHg22+/5cknnwRg0KBBxMVdXt1dheZaYBUVFSQnJzNgwABiYmLYuXMnZ86cYezYsfTr14+xY8dSXl5u9ToKzt7ELF2IQojWeHp6Wr4/duwYS5YsYevWrezfv5/x48e3OEzdxcXF8r2joyMmk6nFY7u6ul5ym6ud5lpgjz32GOPHj2ft2rUYjUbq6up4+eWXGTNmDPPnz2fhwoUsXLiQRYsWWbWOvIp6dDoIkVGIQmjG5baUOkNVVRXe3t5069aNgoICvvjiC8aPH9+hrzFy5EjWrFnDDTfcwIEDB1rsoryaaCrAKisr2bFjB++//z6g/qXi4uLChg0b2L59OwAzZsxg9OjRVg+wgsp6grxdcXbUXCNVCKFBQ4YMITY2lgEDBtC7d29GjhzZ4a/x6KOPct999xEbG2v58vHx6fDXsRc6RVEUWxdxTkZGBikpKcTGxrJv3z6GDh3KkiVLCAsLo6KiAlAvbPr5+Vl+bo1eryc9Pf2ya7nnnz9QazSx/g8d/x+hEKLtjhw5QkxMjK3L0ASTyYTJZMLNzY1jx44xbtw4jh07hpPT5bVFWvrdXulnZ2fSVPPCZDKxd+9eZs+ezU8//YSnpycLFy5sto1Op2v1DvLU1FT0ej16vZ6SkpIrqiW/ol4GcAghNKWmpoaRI0cyaNAgpkyZwrvvvnvZ4dUVaOrMw8PDCQ8PZ8SIEQAkJyezcOFCgoODKSgoIDQ0lIKCAoKCglrcPyUlhZSUFIBmI4PaS1EU8ivrGRPT8usIIYQt+Pr6smfPHluXoRmaaoGFhITQs2dPy5QraWlpxMbGMmHCBFauXAnAypUrmThxolXrKK9rxNBoJlRaYEIIoVmaaoGBeqPe9OnTMRqNREZGsmLFCsxmM1OnTmX58uX07t2bNWvWWLUGWchSCCG0T3MBlpiY2OIFxLS0tE6r4VyAyUz0QgihXZrqQtQKy0KWMo2UEEJolgRYC/IrDbg4OeDv6XLpjYUQXV5SUhJffPFFs8cWL17M7NmzW93Hy8sLgPz8fJKTk1vcZvTo0Zccsr548WLq6uosP996662XvI3oaiEB1oL8inrCfN27xIJvQogrN23aNFavXt3ssdWrVzNt2rRL7tujRw/Wrl172a/96wDbtGkTvr6yzBNIgLVIFrIUQvxScnIyGzdutCxgmZOTQ35+PoMHD2bMmDEMGTKEgQMHsmHDhgv2zcnJIT4+HoD6+nruvvtuYmJimDRpEvX19ZbtZs+ebVmK5bnnngPg9ddfJz8/n6SkJJKSkgCIiIigtLQUgL///e/Ex8cTHx9vWYolJyeHmJgYZs2aRVxcHOPGjWv2Ol2JBFgL8isMMgJRCGHRvXt3hg8fzubNmwG19TV16lTc3d1Zv349e/fuZdu2bTz++ONcbHKjt99+Gw8PD44cOcLzzz/f7J6ul156ifT0dPbv38/XX3/N/v37mTNnDj169GDbtm1s27at2bH27NnDihUr+OGHH9i1axfLli2zLK9y7NgxHn74YQ4dOoSvry8ff/yxFX4rtqe5UYi21thkprhaAkwITdo8HwoPdOwxQwbCLQsvudm5bsSJEyeyevVqli9fjqIoPPXUU+zYsQMHBwfy8vIoKioiJCSkxWPs2LGDOXPmAJCQkEBCQoLluTVr1pCamorJZKKgoIDDhw83e/7Xvv32WyZNmmSZEX/y5Ml88803TJgwgT59+pCYmAg0X46lq5EW2K8UVRkwK9BDuhCFEL8wceJE0tLS2Lt3L3V1dQwdOpQPP/yQkpIS9uzZQ0ZGBsHBwS0uoXIp2dnZvPbaa6SlpbF//35++9vfXtZxzjm3FAt07eVYpAX2K/myDpgQ2tWGlpK1eHl5kZSUxAMPPGAZvFFZWUlQUBDOzs5s27aNU6dOXfQYN954I//7v//LTTfdxMGDB9m/fz+gLsXi6emJj48PRUVFbN68mdGjRwPg7e1NdXU1AQEBzY51ww03cP/99zN//nwURWH9+vV88MEHHX/iGiYB9isFlTILhxCiZdOmTWPSpEmWEYnTp0/n9ttvZ+DAgej1egYMGHDR/WfPns3MmTOJiYkhJiaGoUOHAurqyoMHD2bAgAH07Nmz2VIsKSkpjB8/3nIt7JwhQ4Zw//33M3z4cAAeeughBg8e3GW7C1uiqeVUOtLlLgnw1vbj/PXzLA6/cDMeLpLvQtiaLKdiPbKcShejQ0dUoKeElxBCaJx8Sv/K7NFRzB4dZesyhBBCXIK0wIQQQtglCTAhhOZ10Uv1NtUVfqcSYEIITXNzc6OsrKxLfOBqhaIolJWV4eZm3/e7yjUwIYSmhYeHk5ubS0lJia1L6VLc3NwIDw+3dRlXRHMBFhERgbe3N46Ojjg5OZGens6CBQtYtmwZgYGBALz88svceuutNq5UCNEZnJ2d6dOnj63LEBqkuQAD2LZt2wV3nc+dO5d58+bZqCIhhBBaI9fAhBBC2CXNBZhOp2PcuHEMHTqU1NRUy+NvvPEGCQkJPPDAA5SXl9uwQiGEEFqguamk8vLyCAsLo7i4mLFjx7J06VKio6MJCAhAp9Px7LPPUlBQwHvvvXfBvqmpqZbQy8zMvGBespKSEst1tK6gq50PdL1z6mrnA13vnLra+cCVnVNOTo5lwUyt01yA/dKCBQvw8vJqdu0rJyeH2267jYMHD7b7ePY0x1dbdLXzga53Tl3tfKDrnVNXOx/omufUEk11IdbW1lJdXW35/ssvvyQ+Pp6CggLLNuvXr7cszy2EEOLqpalRiEVFRUyaNAkAk8nE7373O8aPH8+9995LRkYGOp2OiIgI3n33XRtXKoQQwtY0FWCRkZHs27fvgsc7apG2lJSUDjmOVnS184Gud05d7Xyg651TVzsf6Jrn1BJNXwMTQgghWqOpa2BCCCFEW10VAfb5558THR1N3759Wbhwoa3L6RAREREMHDiQxMRE9Hq9rctptwceeICgoKBmA3LOnDnD2LFj6devH2PHjrW7+/1aOqcFCxYQFhZGYmIiiYmJbNq0yYYVts/PP/9MUlISsbGxxMXFsWTJEsB+36fWzsee3yODwcDw4cMZNGgQcXFxPPfccwBkZ2czYsQI+vbty1133YXRaLRxpVaidHEmk0mJjIxUTpw4oTQ0NCgJCQnKoUOHbF3WFevdu7dSUlJi6zIu29dff63s2bNHiYuLszz2pz/9SXnllVcURVGUV155RXniiSdsVd5laemcnnvuOeXVV1+1YVWXLz8/X9mzZ4+iKIpSVVWl9OvXTzl06JDdvk+tnY89v0dms1mprq5WFEVRjEajMnz4cGXnzp3KnXfeqaxatUpRFEX5/e9/r7z11lu2LNNqunwLbPfu3fTt25fIyEhcXFy4++672bBhg63LuurdeOONdO/evdljGzZsYMaMGQDMmDGDTz75xBalXbaWzsmehYaGMmTIEAC8vb2JiYkhLy/Pbt+n1s7Hnul0Ory8vABobGyksbERnU7H1q1bSU5OBuzrPWqvLh9geXl59OzZ0/JzeHi43f9HC61PuWXPioqKCA0NBSAkJISioiIbV9QxusI0aDk5Ofz000+MGDGiS7xPvzwfsO/3qKmpicTERIKCghg7dixRUVH4+vri5KQOMu8qn3kt6fIB1lV9++237N27l82bN/Pmm2+yY8cOW5fUoXQ6HTqdztZlXLHZs2dz4sQJMjIyCA0N5fHHH7d1Se1WU1PDlClTWLx4Md26dWv2nD2+T78+H3t/jxwdHcnIyCA3N5fdu3eTmZlp65I6TZcPsLCwMH7++WfLz7m5uYSFhdmwoo5x7hyCgoKYNGkSu3fvtnFFVy44ONgy60pBQQFBQUE2rujKBQcH4+joiIODA7NmzbK796mxsZEpU6Ywffp0Jk+eDNj3+9Ta+djze3SOr68vSUlJ7Ny5k4qKCkwmE9B1PvNa0uUDbNiwYRw7dozs7GyMRiOrV69mwoQJti7rirQ25Za9mzBhAitXrgRg5cqVTJw40cYVXTl7ngZNURQefPBBYmJi+O///m/L4/b6PrV2Pvb8HpWUlFBRUQFAfX09X331FTExMSQlJbF27VrAvt6jdrP1KJLOsHHjRqVfv35KZGSk8uKLL9q6nCt24sQJJSEhQUlISFBiY2Pt8pzuvvtuJSQkRHFyclLCwsKUf/7zn0ppaaly0003KX379lXGjBmjlJWV2brMdmnpnO655x4lPj5eGThwoHL77bcr+fn5ti6zzb755hsFUAYOHKgMGjRIGTRokLJx40a7fZ9aOx97fo/27dunJCYmKgMHDlTi4uKU559/XlEU9TNi2LBhSlRUlJKcnKwYDAYbV2odMhOHEEIIu9TluxCFEEJ0TRJgQggh7JIEmBBCCLskASaEEMIuSYAJIYSwSxJgQrSTo6OjZebyxMTEDl3hICcnx67uQxLCljS1IrMQ9sDd3Z2MjAxblyHEVU9aYEJ0kIiICJ544gkGDhzI8OHDOX78OKC2qm666SYSEhIYM2YMp0+fBtTJiydNmsSgQYMYNGgQ33//PaBOzjpr1izi4uIYN24c9fX1ALz++uvExsaSkJDA3XffbZuTFEJDJMCEaKf6+vpmXYgfffSR5TkfHx8OHDjAI488wh//+EcAHn30UWbMmMH+/fuZPn06c+bMAWDOnDmMGjWKffv2sXfvXuLi4gA4duwYDz/8MIcOHcLX15ePP/4YgIULF/LTTz+xf/9+3nnnnU4+ayG0R2biEKKdvLy8qKmpueDxiIgItm7dSmRkJI2NjYSEhFBWVkZAQAAFBQU4OzvT2NhIaGgopaWlBAYGkpubi6urq+UYOTk5jB07lmPHjgGwaNEiGhsbeeaZZxg/fjxeXl7ccccd3HHHHZZ1oIS4WkkLTIgO9MulRS53mZFfBpqjo6NlVvGNGzfy8MMPs3fvXoYNG2Z5XIirlQSYEB3oXHfiRx99xLXXXgvAddddx+rVqwH48MMPueGGGwAYM2YMb7/9NqBe96qsrGz1uGazmZ9//pmkpCQWLVpEZWVli61AIa4mMgpRiHY6dw3snPHjx1uG0peXl5OQkICrqyurVq0CYOnSpcycOZNXX32VwMBAVqxYAcCSJUtISUlh+fLlODo68vbbb1tWOv61pqYm7rnnHiorK1EUhTlz5uDr62vlMxVC2+QamBAdJCIigvT0dAICAmxdihBXBelCFEIIYZekBSaEEMIuSQtMCCGEXZIAE0IIYZckwIQQQtglCTAhhBB26f8BW7/AzHFwC10AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "image/png": {
       "height": 300
      }
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Image\n",
    "Image(save_dir+'/accu_history_class.png', height=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "!conda env export > environment.yml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyMr9K8NdlsV41UO8GEL8I1d",
   "collapsed_sections": [
    "eRPoSx89oDiy",
    "n0_2EFDHhvfz",
    "6s_zy_WIfUP5"
   ],
   "name": "train_CO_tips.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}