figure_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "from models.model_resnet import *\n",
    "from models.model_resnet18 import *\n",
    "import myData.iDataset\n",
    "import myData.iDataLoader\n",
    "from utils import *\n",
    "from sklearn.utils import shuffle\n",
    "import trainer.trainer_warehouse\n",
    "import trainer.evaluator\n",
    "from myData.data_warehouse import *\n",
    "import easydict\n",
    "from models.W_resnet import *\n",
    "from sklearn.metrics import pairwise_distances\n",
    "import pickle\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = easydict.EasyDict({\"batch_size\": 128,\n",
    "                          \"replay_batch_size\": 64,\n",
    "                          \"lr\": 0.1,\n",
    "                          \"bft_lr\" : 0.01,\n",
    "                          \"schedule\": [60,80],\n",
    "                          \"bft_schedule\": [30,40],\n",
    "                          \"gammas\" : [0.1, 0.1,0.1],\n",
    "                          \"momentum\" : 0.9,\n",
    "                          \"seed\" : 1,\n",
    "                          \"decay\" : 0.0005,\n",
    "                          \"step_size\" : 10,\n",
    "                          \"nepochs\" : 100,\n",
    "                          \"workers\" : 0,\n",
    "                          \"start_classes\" : 10,\n",
    "                          \"memory_size\" : 2000,\n",
    "                          \"dataset\" : \"CIFAR10\",\n",
    "                          \"device\" : \"cuda\",\n",
    "                          \"KD\" : \"naive_global\",\n",
    "                          \"trainer\" : 'vanilla', \n",
    "                          \"CCtriplet\" : False,\n",
    "                          \"distance\" : \"Eucledian\",\n",
    "                          \"beta\" : 1,\n",
    "                          \"margin\" : 0.5,\n",
    "                          \"triplet_lam\" : 0.5,\n",
    "                          \"dict_type\" : \"softmax\",\n",
    "                          \"dict_update\" : True,\n",
    "                          \"new_WA\" : True,\n",
    "                          \"triplet_epoch\" : 10,\n",
    "                          \"anchor_update_epoch\" : 10,\n",
    "                          \"GPU_NUM\" : 0,\n",
    "                          \"model\" : \"resnet32\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./checkpoint/figure_model/base_vanilla_CIFAR10_triFalse_5_4_resnet32.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hongjun\\git_repository\\CL_sampling\\models\\model_resnet.py:92: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  init.kaiming_normal(m.weight)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CifarResNet(\n",
       "  (conv_1_3x3): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "  (bn_1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (stage_1): Sequential(\n",
       "    (0): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (1): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (4): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (stage_2): Sequential(\n",
       "    (0): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): DownsampleA(\n",
       "        (avg): AvgPool2d(kernel_size=1, stride=2, padding=0)\n",
       "      )\n",
       "    )\n",
       "    (1): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (4): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (stage_3): Sequential(\n",
       "    (0): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(32, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (downsample): DownsampleA(\n",
       "        (avg): AvgPool2d(kernel_size=1, stride=2, padding=0)\n",
       "      )\n",
       "    )\n",
       "    (1): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (2): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (3): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (4): ResNetBasicblock(\n",
       "      (conv_a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_a): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (conv_b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn_b): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AvgPool2d(kernel_size=8, stride=8, padding=0)\n",
       "  (fc): Linear(in_features=64, out_features=10, bias=True)\n",
       "  (gate_fc): Linear(in_features=64, out_features=5, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models_dir = \"./checkpoint/figure_model/\"\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "total_task = 5\n",
    "tasknum = 4\n",
    "\n",
    "model_dir = os.path.join(models_dir, 'base_{}_{}_tri{}_{}_{}_{}.pt'.format(args.trainer, args.dataset,args.CCtriplet, total_task, tasknum, args.model))    \n",
    "print(model_dir)\n",
    "saved_model = torch.load(model_dir, map_location='cuda:0')\n",
    "numclass=10\n",
    "\n",
    "net = resnet32(num_classes=numclass, tasknum=total_task)\n",
    "net.load_state_dict(saved_model)\n",
    "\n",
    "net.eval()\n",
    "net.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hongjun\\Anaconda3\\envs\\pytorch_project\\lib\\site-packages\\torch\\cuda\\memory.py:346: FutureWarning: torch.cuda.memory_cached has been renamed to torch.cuda.memory_reserved\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current cuda :  0\n",
      "GeForce RTX 2070 SUPER\n",
      "Memory Usage:\n",
      "Allocated: 0.0 GB\n",
      "Cached:    0.0 GB\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "\n",
    "GPU_NUM = 0 # 원하는 GPU 번호 입력\n",
    "device = torch.device(f'cuda:{GPU_NUM}' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "torch.cuda.set_device(device)\n",
    "print(\"current cuda : \", torch.cuda.current_device())\n",
    "\n",
    "if device.type == 'cuda':\n",
    "    print(torch.cuda.get_device_name(GPU_NUM))\n",
    "    print('Memory Usage:')\n",
    "    print('Allocated:', round(torch.cuda.memory_allocated(GPU_NUM)/1024**3,1), 'GB')\n",
    "    print('Cached:   ', round(torch.cuda.memory_cached(GPU_NUM)/1024**3,1), 'GB')\n",
    "\n",
    "\n",
    "\n",
    "#seed\n",
    "seed = args.seed\n",
    "set_seed(seed)\n",
    "\n",
    "data = DatasetWH\n",
    "dataset = data.get_dataset(args.dataset)\n",
    "tasknum = (dataset.classes - args.start_classes) // args.step_size + 1\n",
    "\n",
    "\n",
    "#shuffle_idx = shuffle(np.arange(dataset.classes), random_state = 1)\n",
    "shuffle_idx = None\n",
    "\n",
    "test_dataset_loader = myData.iDataLoader.IncrementalLoader(dataset.test_data,\n",
    "                                                            dataset.test_labels,\n",
    "                                                            dataset.classes,\n",
    "                                                            args.step_size,\n",
    "                                                            args.memory_size,\n",
    "                                                            'test',\n",
    "                                                            transform=dataset.test_transform,\n",
    "                                                            loader=None,\n",
    "                                                            shuffle_idx=shuffle_idx,\n",
    "                                                            base_classes=args.start_classes,\n",
    "                                                            approach= \"wa\"\n",
    "                                                            )\n",
    "\n",
    "\n",
    "test_iterator = torch.utils.data.DataLoader(test_dataset_loader, batch_size=100, shuffle=False)\n",
    "\n",
    "testType = \"trainedClassifier\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "target_list = []\n",
    "\n",
    "class_prob = []\n",
    "def one_epoch_test(testloader=None, current_epoch=None):\n",
    "    global best_acc\n",
    "    global best_class_acc\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    print('\\nEpoch: %d' % current_epoch)\n",
    "    print(\"Testing...\")\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    class_correct = 0\n",
    "    class_acc = []\n",
    "    prob = 0\n",
    "    #class_prob = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, targets, traindata_idx in tqdm(testloader):\n",
    "            inputs = inputs.float().cuda()\n",
    "            inputs, targets = inputs.to(device), targets.to(device)\n",
    "            outputs = net(inputs)[:, :args.start_classes]\n",
    "\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "\n",
    "            class_correct += predicted.eq(targets).sum().item()\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "            prob += torch.mean(F.softmax(outputs, dim=1), dim=0)\n",
    "\n",
    "            acc = correct / total\n",
    "\n",
    "        if acc > best_acc:\n",
    "            best_acc = acc\n",
    "            best_class_acc = class_acc\n",
    "            print('Saving..')\n",
    "            state = {\n",
    "                'net': net.state_dict(),\n",
    "                'acc': acc,\n",
    "                'epoch': current_epoch,\n",
    "            }\n",
    "\n",
    "        print(best_acc)\n",
    "        \n",
    "def run_featureMap(testloader = None, model = None, current_epoch=None) :\n",
    "    #print('\\nEpoch: %d' % current_epoch)\n",
    "    print(\"Testing...\")\n",
    "    \n",
    "    model = model.cuda()\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    class_correct = 0\n",
    "    class_acc = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for inputs, targets, traindata_idx in testloader :\n",
    "            inputs = inputs.float().cuda()\n",
    "            targets = targets.type(dtype=torch.long).cuda()\n",
    "\n",
    "            outputs, features = model(inputs, feature_return = True)\n",
    "\n",
    "            \n",
    "            targets, features = targets.cpu(), features.cpu()\n",
    "            target_list.append(targets)\n",
    "            feature_output_list.append(features)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_avg_feature_map(net) :\n",
    "    best_class_acc = 0\n",
    "    best_acc = 0\n",
    "\n",
    "\n",
    "    for i in range(1) :\n",
    "        run_featureMap(test_iterator, net, i)\n",
    "\n",
    "    featureMap = np.concatenate(feature_output_list)\n",
    "    targetList = np.concatenate(target_list)\n",
    "\n",
    "    class_feature = []\n",
    "    avg_feature = []\n",
    "    std_feature = []\n",
    "\n",
    "    for i in range(args.start_classes) :\n",
    "        class_feature.append(featureMap[i*1000:(i+1)*1000])\n",
    "        temp_mean = np.mean(class_feature[i], axis=0)\n",
    "        #temp_std = np.std(class_feature[i], axis=0)\n",
    "        avg_feature.append(torch.tensor(temp_mean))\n",
    "        #std_feature.append(torch.tensor(temp_std))\n",
    "\n",
    "    classnum = args.start_classes\n",
    "    cos = torch.nn.CosineSimilarity(dim=0)\n",
    "\n",
    "    mtx = np.zeros((classnum,classnum))\n",
    "    \n",
    "    for i in range(classnum) :\n",
    "        for j in range(classnum) :\n",
    "            mtx[i,j] = cos(avg_feature[i], avg_feature[j])\n",
    "    \n",
    "    return mtx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "global feature_output_list\n",
    "feature_output_list = []\n",
    "mtx = get_avg_feature_map(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot CIFAR-10 with r+KD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAHWCAYAAABDtELCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3debRld1Un8O+uSkIIhEFACalooDsgwYEhBhUbUESD2tBxaIO2AgsptQmgtnZDdy9UtFtRG5u1wKGQIE4ECQ6FZgGCgqAMKUKCZCQElEoEQhRBjZLK2/3Hu7EfL1XvvlveW+ee+z6ftc6qO5z7O/u+VKp27f37/U51dwAAlsGuoQMAALidxAQAWBoSEwBgaUhMAIClITEBAJaGxAQAWBoSEwDgqFTVOVV1TVVdV1XPPcz7X1BVb66q91XVW6pqz9Qxp+1jUlVfmORJSU5N0kluTLK/u686uq8BAIxdVe1Ocm2Sxyc5mOSSJE/u7is3nPOaJH/Q3a+sqq9J8rTu/q6txt2yYlJV/y3JhUkqybsnF60krzpcZgQA7BhnJ7muu6/v7s9kPV940qZzzkzy5snjPznM+3dw3JT3n57kId1968YXq+pFSa5I8tPbCBwAWD2nJvnIhucHkzxy0zmXJ/mWJC9Ocm6Sk6vqXt1985EGnZaYrCW5X5K/3PT6KZP3Dquq9ibZmyTn7/rcR5yz6x5TLrM8HnLV64YOYWb3ff/rhw5hZlee+Z1DhzCTXXXE3+5L6x0fvM/QIczsQx++ZegQZnLuY26dftKSOe2lTx86hJlc//2/OnQIR+XRD7lLHcvr/eHxD5r7/WW+6dC135vJ3+UT+7p734bnh/uOm+P44SQvqaqnJvnTJDckObTVdaclJj+Q5M1V9YH8/6zo85P82yTnH+lDk8D3JYv5YQEAi7Xx7/IjOJjktA3P92R9HurGMW5M8s1JUlV3TfIt3f13W113y8Sku19fVQ/Meh/p1KxnRweTXNLdt231WQDg2Kjjj2mB5naXJDmjqu6f9UrIeUm+47Piqrp3kr/p7rUkz0tywbRBp1VMMhnsnUcTMQCwmrr7UFWdn+QNSXYnuaC7r6iqFyQ50N37kzw2yU9VVWe9lfPMaeNOTUwAgOW267hBKibp7ouTXLzptedveHxRkotmGdMGawDA0lAxAYCRq+NXp84gMQGAkRuqlbMIq5NiAQCjp2ICACM30HLhhVAxAQCWhooJAIzcKs0xkZgAwMhp5QAALICKCQCM3Cq1clRMAICloWICACNXu1enYiIxAYCR27VCiYlWDgCwNFRMAGDkapeKCQDA3KmYAMDI1e7VqTOszjcBAEZPxQQARm6VVuUsPDF5yFWvW/Ql5uqKB//7oUOY2YG3XzV0CDO7+cPjKta9+103DR3CzE46+VNDhzCzE07YPXQIM/nop+8ydAgzO+5ZLxs6hJk8+NfPHzqEo/OTrzimlzP5FQBgAbRyAGDkVqmVo2ICACwNFRMAGDn3ygEAlkbtWp0GyOp8EwBg9FRMAGDkLBcGAFgAFRMAGLlVWi4sMQGAkdPKAQBYABUTABg5y4UBABZAxQQARm6V5phITABg5FZpVY5WDgCwNFRMAGDkVqmVo2ICACwNFRMAGDnLhQEAFkDFBABGbpXmmEhMAGDkVikxOepWTlU9bZ6BAAD8a+aY/PiR3qiqvVV1oKoO/NaFr/5XXAIAmKZ21dyPoWzZyqmq9x3prSSfd6TPdfe+JPuS5MPXXdtHHR0AsKNMm2PyeUm+Psnfbnq9kvz5QiICAGaySsuFpyUmf5Dkrt192eY3quotC4kIAJjJKt0rZ8vEpLufvsV73zH/cACAncxyYQAYOcuFAQAWQMUEAEZuJ01+BQCWnFYOAMACqJgAwMipmAAALICKCQCM3CpNfl2dbwIAjJ6KCQCM3CrNMZGYAMDIaeUAACyAigkAjF2tTitHxQQAWBoqJgAwcia/AgBLw+RXAIAFUDEBgJFbpVaOigkAsDQWXjG57/tfv+hLzNWBt181dAgzu/NXPXjoEGb20LPuMXQIM/myk44fOoSZ3fMLPmfoEGZ2t9PvO3QIMznxtm8YOoSZXV2PHjqEmfzsnl8YOoSj8jPH+HqrNMdEKwcARk4rBwBgAVRMAGDkVEwAABZAxQQAxm6FJr+uzjcBAEZPxQQARq5W6O7CEhMAGLlV2sdkdb4JADB6KiYAMHKWCwMAO15VnVNV11TVdVX13MO8//lV9SdV9d6qel9VTb2Pg4oJAIzdAHNMqmp3kpcmeXySg0kuqar93X3lhtP+Z5Lf7u5frKozk1yc5PStxpWYAMDIDdTKOTvJdd19fZJU1YVJnpRkY2LSSe42eXz3JDdOG1RiAgAcjVOTfGTD84NJHrnpnB9L8saqelaSuyT52mmDmmMCACNXtWsBR+2tqgMbjr2bL3uYUHrT8ycn+dXu3pPkG5L8elVtmXuomAAAd9Dd+5Ls2+KUg0lO2/B8T+7Yqnl6knMm472jqk5Mcu8kHz/SoComADB2u2r+x3SXJDmjqu5fVSckOS/J/k3n/FWSxyVJVT04yYlJbtpqUBUTABi5IXZ+7e5DVXV+kjck2Z3kgu6+oqpekORAd+9P8l+SvKyqfjDrbZ6ndvfmds9nkZgAAEeluy/O+hLgja89f8PjK5M8apYxJSYAMHJ2fgUAWICpiUlVfWFVPa6q7rrp9XMWFxYAsG21a/7HQLa8clU9O8nvJ3lWkvdX1ZM2vP2/FxkYALDzTJtj8owkj+juv6+q05NcVFWnd/eLc/iNVQCAY2wnzTHZ3d1/nyTd/eEkj03yhKp6UbZITDbuFvfyN/75vGIFAA5n1675H0N9lSnvf7SqHnr7k0mS8k1Z37Xti4/0oe7e191ndfdZT/+6r5xPpADAypvWyvnuJIc2vtDdh5J8d1X98sKiAgC2rWp1WjlbJibdfXCL9/5s/uEAADuZDdYAYOwGnBMybxITABi5nbQqBwDgmFExAYCxG3Cn1nlbnW8CAIyeigkAjN0KzTGRmADAyJVWDgDA/KmYAMDYrVArR8UEAFgaKiYAMHK1Qju/rs43AQBGT8UEAMZup9xdGAAYAa0cAID5UzEBgLFboVaOigkAsDRUTABg5FZpubDEBADGboXulbPwxOTKM79z0ZeYq5s/PL7/uA896x5DhzCzTxz45NAhzORh5z986BBm9t6XXDp0CDP70u87cegQZnKny981dAgze/N9njh0CDNZ638aOgSOMRUTABg798oBAJg/FRMAGLkyxwQAWBpaOQAA86diAgBjt0KtnNX5JgDA6KmYAMDYuVcOAMD8qZgAwNi5Vw4AsDRMfgUAmD8VEwAYOxusAQDMn4oJAIzdCs0xkZgAwNjZxwQAYP5UTABg7FZoH5PV+SYAwOipmADA2K3QHBOJCQCM3U5alVNVZyfp7r6kqs5Mck6Sq7v74oVHBwDsKFsmJlX1o0mekOS4qvqjJI9M8pYkz62qh3X3/1p8iADAllZo8uu0ism3Jnlokjsl+WiSPd39qar62STvSiIxAQDmZlqKdai7b+vuf0zywe7+VJJ09y1J1o70oaraW1UHqurA77z6lXMMFwC4g6r5HwOZVjH5TFWdNElMHnH7i1V192yRmHT3viT7kuTSa2/ueQQKAKy+aYnJo7v7n5OkuzcmIscnecrCogIAtm+nrMq5PSk5zOufSPKJhUQEAMxmhfYxWZ0UCwAYPRusAcDYrdBy4dX5JgDA6KmYAMDI9QrNMZGYAMDYrdCqnNX5JgDA6KmYAMDYqZgAAMyfigkAjJzJrwDA8tDKAQCYPxUTABi7FWrlqJgAAEtDxQQAxm6F7pUjMQGAkVulVTmrk2IBAKOnYgIAY2e5MADA/KmYAMDItYoJAMD8LbxisqvWFn2JuXr3u24aOoSZfdlJxw8dwswedv7Dhw5hJu99yaVDhzCzx7z43KFDmNmuE+80dAgz+eePje/Piys/Oq6YP3HDzUOHcJTOPLaXsyoHAFgWXbvmfmxHVZ1TVddU1XVV9dzDvP/zVXXZ5Li2qj45bUxzTACAmVXV7iQvTfL4JAeTXFJV+7v7ytvP6e4f3HD+s5I8bNq4KiYAMHZV8z+mOzvJdd19fXd/JsmFSZ60xflPTvKqaYNKTACAo3Fqko9seH5w8todVNUXJLl/kj+eNqhWDgCM3QKWC1fV3iR7N7y0r7v3bTzlMB/rIwx3XpKLuvu2adeVmADAyC3iXjmTJGTfFqccTHLahud7ktx4hHPPS/LM7VxXKwcAOBqXJDmjqu5fVSdkPfnYv/mkqnpQknsmecd2BlUxAYCxG2Dn1+4+VFXnJ3lDkt1JLujuK6rqBUkOdPftScqTk1zY3Udq83wWiQkAcFS6++IkF2967fmbnv/YLGNKTABg5Pqw81DHSWICACPnJn4AAAugYgIAY6diAgAwfyomADByi9hgbSgqJgDA0lAxAYCRW6VVORITABg7rRwAgPlTMQGAkVulVs7qfBMAYPRmTkyq6tcWEQgAcHQ6NfdjKFu2cqpq/+aXknx1Vd0jSbr7iYsKDADYnp3UytmT5FNJXpTk/0yOT294fFhVtbeqDlTVgddeqMACAGzPtMmvZyV5TpL/keRHuvuyqrqlu9+61Ye6e1+SfUly2Qdu6rlECgAc3gotF94yMenutSQ/X1Wvmfz6sWmfAQA4WttKMrr7YJJvq6pvzHprBwBYEr1Ci2xnqn509x8m+cMFxQIAHAU38QMAWADzRQBg5HbScmEAgGNGxQQARm7InVrnTcUEAFgaKiYAMHKrNMdEYgIAI2e5MADAAqiYAMDImfwKALAAKiYAMHImvwIAS0MrBwBgAVRMAGDkVqmVszrfBAAYPRUTABi5VZpjIjEBgJHTygEAWAAVEwAYOa2cGbzjg/dZ9CXm6qSTPzV0CDO75xd8ztAhzOy9L7l06BBm8pgXnzt0CDN763N+d+gQZva4C79v6BBmUjfdPHQIM7vhAweHDmEmpzzgfkOHwDGmYgIAI+fuwgAAC6BiAgAj1706FROJCQCMXK9QA2R1vgkAMHoqJgAwcqu0XFjFBABYGiomADByq1QxkZgAwMitUmKilQMALA0VEwAYORUTAIAFUDEBgJGz8ysAsDS0cgAAFkDFBABGTsUEAGABVEwAYORUTAAAFkDFBABGznJhAGBprK1QK2emxKSqvirJ2Une391vXExIAMBOteUck6p694bHz0jykiQnJ/nRqnrugmMDALahU3M/hjJt8uvxGx7vTfL47v7xJF+X5DsXFhUAsCNNS0x2VdU9q+peSaq7b0qS7v6HJIeO9KGq2ltVB6rqwNsv3jfHcAGAzbpr7sdQps0xuXuS9ySpJF1V9+3uj1bVXSevHVZ370uyL0l+8fXpeQULANzRKu1jsmVi0t2nH+GttSTnzj0aAGBHO6rlwt39j0k+NOdYAICjsEr7mNj5FQBYGjZYA4CR2zFzTACA5aeVAwCwAComADBya0MHMEcqJgDA0lAxAYCRM8cEAGABVEwAYOQsFwYAloZWDgDAAkhMAGDkOjX3Yzuq6pyquqaqrquq5x7hnP9YVVdW1RVV9VvTxtTKAQBmVlW7k7w0yeOTHExySVXt7+4rN5xzRpLnJXlUd/9tVX3utHElJgAwcms9yGXPTnJdd1+fJFV1YZInJblywznPSPLS7v7bJOnuj08bVCsHAEZuoFbOqUk+suH5wclrGz0wyQOr6s+q6p1Vdc60QVVMAIA7qKq9SfZueGlfd+/beMphPra5dnNckjOSPDbJniRvq6ov6u5PHum6EhMAGLlFLBeeJCH7tjjlYJLTNjzfk+TGw5zzzu6+NcmHquqarCcqlxxpUK0cAOBoXJLkjKq6f1WdkOS8JPs3nfN7Sb46Sarq3llv7Vy/1aAqJgAwcj3A5NfuPlRV5yd5Q5LdSS7o7iuq6gVJDnT3/sl7X1dVVya5LcmPdPfNW4278MTkQx++ZdGXmKsTTtg9dAgzu9vp9x06hJl96fedOHQIM9l14p2GDmFmj7vw+4YOYWZvPu+Xhg5hJje/6ZqhQ5jZLW9/69AhzOT6y64dOoSjdMoxvdraQFvSd/fFSS7e9NrzNzzuJD80ObZFKwcAWBpaOQAwcu6VAwCwAComADByQ0x+XRQVEwBgaaiYAMDIbfduwGMgMQGAkRvoJn4LoZUDACwNFRMAGDnLhQEAFkDFBABGbpWWC0tMAGDkhrpXziJo5QAAS0PFBABGbpVaOSomAMDSUDEBgJFbpeXCEhMAGDk7vwIALICKCQCMnMmvAAALoGICACPXO2WDtap6ZFXdbfL4zlX141X1uqp6YVXd/diECADsFNNaORck+cfJ4xcnuXuSF05ee8UC4wIAtmmt538MZVorZ1d3H5o8Pqu7Hz55/PaqumyBcQEA27STJr++v6qeNnl8eVWdlSRV9cAktx7pQ1W1t6oOVNWBy9/28jmFCgCsummJyfckeUxVfTDJmUneUVXXJ3nZ5L3D6u593X1Wd5/1pf/u6fOLFgC4g+75H0PZspXT3X+X5KlVdXKSB0zOP9jdHzsWwQEAO8u2lgt396eTXL7gWACAo7DmXjkAwLLYSZNfAQCOGRUTABg5FRMAgAVQMQGAkRtyp9Z5k5gAwMj1Cq3K0coBAJaGigkAjJzJrwAAC6BiAgAjZ/IrALA0tHIAABZAxQQARk7FBABgAVRMAGDkVmnyq4oJALA0VEwAYORWaY6JxAQARm5tbegI5kcrBwBYGiomADByWjkzOPcxty76EnP10U/fZegQZnbibd8wdAgzu9Pl7xo6hJn888duGjqEmdVNNw8dwsxuftM1Q4cwk3t97YOGDmFm577m6qFDmMlZZ9wydAgcYyomADByKiYAwNKwjwkAwAKomADAyPVCejm1gDGnUzEBAJaGigkAjJzJrwDA0rDzKwDAAqiYAMDIrVIrR8UEAFgaKiYAMHI2WAMAWAAVEwAYuVWaYyIxAYCR64X0cuz8CgDscComADByJr8CACyAigkAjJzJrwDA0lhboV6OVg4AsDRUTABg5FaplbNlxaSqnl1Vpx2rYACAnW1aK+cnkryrqt5WVf+5qu5zLIICALave/7HUKYlJtcn2ZP1BOURSa6sqtdX1VOq6uQjfaiq9lbVgao68Hu//Yo5hgsAbLbWPfdjKNPmmHR3ryV5Y5I3VtXxSZ6Q5MlJfi7JYSso3b0vyb4kecdVn1qhzhcAsEjTEpPP2ii/u29Nsj/J/qq688KiAgC2rdeGjmB+prVyvv1Ib3T3LXOOBQDY4basmHT3tccqEADg6PQKrRe2wRoAsDQkJgAwcmtr8z+2o6rOqaprquq6qnruYd5/alXdVFWXTY7vmTamnV8BYOSGaOVU1e4kL03y+CQHk1xSVfu7+8pNp766u8/f7rgqJgDA0Tg7yXXdfX13fybJhUme9K8dVGICACO31vM/tuHUJB/Z8Pzg5LXNvqWq3ldVF23nNjcSEwDgDjbu4j459m4+5TAf25zSvC7J6d39JUnelOSV065rjgkAjFxvs8Qx05gbdnE/goNJNlZA9iS5cdMYN294+rIkL5x2XRUTABi5gW7id0mSM6rq/lV1QpLzsr47/L+oqlM2PH1ikqumDapiAgDMrLsPVdX5Sd6QZHeSC7r7iqp6QZID3b0/ybOr6olJDiX5myRPnTauxAQARm5tAa2c7ejui5NcvOm15294/Lwkz5tlTK0cAGBpqJgAwMit0r1yJCYAMHK9zS3kx0ArBwBYGiomADByayvUylExAQCWhooJAIzcKk1+VTEBAJaGigkAjNxQG6wtwsITk9Ne+vRFX2KujnvWy4YOYWZX16OHDmFmb77PE4cOYSZXfvSmoUOY2Q0fODh0CDO75e1vHTqEmZz7mquHDmFmX/htXzh0CDN54F+8eugQjtJDj+nVVqiTo5UDACwPrRwAGLleoVaOigkAsDRUTABg5FZpgzWJCQCMnFYOAMACqJgAwMipmAAALICKCQCM3AoVTCQmADB2WjkAAAugYgIAI9crtI+JigkAsDRUTABg5NbMMQEAmD8VEwAYuVWaYyIxAYCRs1wYAGABVEwAYORUTAAAFkDFBABGbm2nTH6tqhOSnJfkxu5+U1V9R5KvTHJVkn3dfesxiBEA2MIqtXKmVUxeMTnnpKp6SpK7JvmdJI9LcnaSpyw2PABgJ5mWmHxxd39JVR2X5IYk9+vu26rqN5JcfqQPVdXeJHuT5Kcf+/B850MeMLeAAYDPtkr7mEyb/Lpr0s45OclJSe4+ef1OSY4/0oe6e193n9XdZ0lKAIDtmlYxeXmSq5PsTvI/krymqq5P8uVJLlxwbADANqzSvXK2TEy6++er6tWTxzdW1a8l+dokL+vudx+LAAGAre2kya/p7hs3PP5kkosWGhEAsGPZxwQARm4nTX4FADhmVEwAYOR6bW3oEOZGxQQAWBoqJgAwcjtmuTAAsPxMfgUAWAAVEwAYuVXaYE3FBABYGiomADByq1QxkZgAwMittX1MAADmTsUEAEZulVo5KiYAwNJQMQGAkVulionEBABGzs6vAAALoGICACO3tma5MADA3C28YnL99//qoi8xVw/+9fOHDmFmP7vnF4YOYWZr/U9DhzCTT9xw89AhzOyUB9xv6BBmdv1l1w4dwkzOOuOWoUOY2QP/4tVDhzCTS7/424cO4ah8463XHNPrrdLkVxUTAGBpmGMCACPXK7QlvcQEAEZOKwcAYAFUTABg5FRMAAAWQMUEAEZuzeRXAGBZaOUAACyAigkAjFy7Vw4AwPypmADAyK3SHBOJCQCM3CptSa+VAwAsDRUTABi5tRVq5aiYAABLQ8UEAEbOcmEAgAVQMQGAkbNcGABYGpYLAwA7XlWdU1XXVNV1VfXcLc771qrqqjpr2phTKyZV9W+SnJvktCSHknwgyau6++9miB0AWJAhWjlVtTvJS5M8PsnBJJdU1f7uvnLTeScneXaSd21n3C0rJlX17CS/lOTEJF+W5M5ZT1DeUVWPnfE7AACr4+wk13X39d39mSQXJnnSYc77iSQ/k+SftjPotIrJM5I8tLtvq6oXJbm4ux9bVb+c5PeTPGzb4QMACzHQcuFTk3xkw/ODSR658YSqeliS07r7D6rqh7cz6HYmvx6X5LYkd0pycpJ0919V1fFH+kBV7U2yd/L0e7t733aCmVVV7Z372D/5irkOt9FC4s16Grooi4p5URYX75nzH3JibD/jZJExnzL/IeNn/NkeOv8hJxYR8zfees08h/ssY/x9cSRvf91jat5jbvq7PEn2bfp5He6a/9JTqqpdSX4+yVNnum73kftSVfWcJE9P8s4kj07ywu5+RVXdJ8lru/vRs1xs3qrqQHdPnUizLMYWbzK+mMcWbyLmY2Fs8SZiPhbGFu+yqaqvSPJj3f31k+fPS5Lu/qnJ87sn+WCSv5985L5J/ibJE7v7wJHG3bJi0t0vrqo3JXlwkhd199WT12/KeqICAOxMlyQ5o6run+SGJOcl+Y7b35wskrn37c+r6i1JfnirpCTZRiunu69IcsXRxQwArKLuPlRV5yd5Q5LdSS7o7iuq6gVJDnT3/qMZd+wbrI2tNzi2eJPxxTy2eBMxHwtjizcR87EwtniXTndfnOTiTa89/wjnPnY7Y245xwQA4Fiy8ysAsDRGmZhsdwvcZVFVF1TVx6vq/UPHsh1VdVpV/UlVXVVVV0xWZy21qjqxqt5dVZdPYv7xoWPajqraXVXvrao/GDqW7aiqD1fVX1TVZVW15QS2ZVFV96iqi6rq6snv6a8YOqatVNWDJj/f249PVdUPDB3XVqrqByf/372/ql5VVScOHdM0VfWcSbxXLPvPd6cZXStnsgXutdmwBW6SJ2/eAneZVNWjs75c6te6+4uGjmeaqjolySndfelkK+H3JPkPS/4zriR36e6/n+yx8/Ykz+nudw4c2paq6oeSnJXkbt39TUPHM01VfTjJWd39iaFj2a6qemWSt3X3r1TVCUlO6u5PDh3Xdkz+vLshySO7+y+HjudwqurUrP//dmZ331JVv531zTh/ddjIjqyqvijru5SeneQzSV6f5Pu7+wODBkaScVZMtrsF7tLo7j/N+trtUejuv+7uSyePP53kqqzv8Le0et3ta+WPnxxLnXVX1Z4k35jkV4aOZVVV1d2yvrXBy5Okuz8zlqRk4nFJPrisSckGxyW5c1Udl+SkJDcOHM80D07yzu7+x+4+lOStWb8nHEtgjInJ4bbAXeq/NMesqk7P+q0HtnXzpSFN2iKXJfl4kj/q7mWP+f8m+a9JxnS/8k7yxqp6z2RXyGX3gCQ3JXnFpGX2K1V1l6GDmsF5SV41dBBb6e4bkvxckr9K8tdJ/q673zhsVFO9P8mjq+peVXVSkm/I+n3gWAJjTEy23AKX+amquyZ5bZIf6O5PDR3PNN19W3c/NMmeJGdPyrVLqaq+KcnHu/s9Q8cyo0d198OTPCHJMydtymV2XJKHJ/nF7n5Ykn9IsvTz0pJk0nZ6YpLXDB3LVqrqnlmvWt8/yf2S3KWq/tOwUW2tu69K8sIkf5T1Ns7lSQ4NGhT/YoyJycF8dma7J8tfNhydyTyN1yb5ze7+naHjmcWkVP+WJOcMHMpWHpXkiZM5Gxcm+Zqq+o1hQ5quu2+c/PrxJL+b9dbqMjuY5OCG6tlFWU9UxuAJSS7t7o8NHcgUX5vkQ919U3ffmuR3knzlwDFN1d0v7+6HT26t8jdJzC9ZEmNMTP5lC9zJvyjOS3JUu8txeJOJpC9PclV3v2joeLajqu5TVfeYPL5z1v+wvHrYqI6su5/X3Xu6+/Ss/x7+4+5e6n9lVtVdJpOhM2mHfF3WS+JLq7s/muQjVfWgyUuPS7K0k7g3eXKWvI0z8VdJvryqTpr82fG4rM9LW2pV9bmTXz8/yTdnHD/rHWF0O78eaQvcgcPaUlW9Ksljk9y7qg4m+dHufvmwUW3pUUm+K8lfTOZsJMl/n+zwt6xOSfLKySqGXUl+u7tHsQR3RD4vye+u/92T45L8Vne/ftiQtuVZSX5z8g+Z65M8beB4pprMe3h8ktQMswAAAABeSURBVO8dOpZpuvtdVXVRkkuz3g55b8axo+prq+peSW5N8szu/tuhA2Ld6JYLAwCra4ytHABgRUlMAIClITEBAJaGxAQAWBoSEwBgaUhMAIClITEBAJaGxAQAWBr/DziypbNC5HW+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mp.subplots(figsize=(10,8))\n",
    "sns.heatmap(mtx, annot=False, annot_kws={'size':20}, cmap=\"coolwarm\", vmin=0.4, vmax=0.9) #dist_out : class_matrix\n",
    "\n",
    "mp.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot CIFAR-10 with joint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing...\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAHWCAYAAABDtELCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3debSkd1kn8O/TnbBjUHaTYCIEAgMYIBMUGAQRDMqBEVQCowIHaY9DAJ3BMeicKHhmHBwHBkeO2kgQRiBIAG0hByKbEmRJA2HJBiEiaSIkyr5JOv3MH7fCXDrdt261Vf3WW/fzOadO1/LWW0/d08vT39/yVncHAGAZbBu6AACA62lMAICloTEBAJaGxgQAWBoaEwBgaWhMAICloTEBAA5JVZ1aVZdV1eVVdcYBXv++qnpbVX2kqt5ZVcdMPee0fUyq6sQkj0lydJJOclWSXd19yaF9DQBg7Kpqe5KPJ3l4kj1JLkjyhO6+eN0xr03yxu5+eVX9SJKndPfPbXTeDROTqvq1JGcnqSTvn3xoJXn1gTojAGDLOCXJ5d19RXd/K2v9wmP2O+YeSd42uf+OA7x+A0dMef2pSf5Nd1+7/smqekGSi5L8j00UDgCsnqOTXLnu8Z4k99/vmA8neVySFyX5ySS3rKpbd/c/H+yk0xqTfUm+N8k/7Pf8HSevHVBV7UiyI0lO33a7+5267VZTPmZ5XH3eZUOXMLNH3e59Q5cws/O/dvLQJczkc1/YPnQJM/vYRV8auoSZXXPVF4cuYSYPeuidhi5hZj995wuHLmEmb73mpKFLOCQ/+++qDufnvenIu839+jKP2vvxX8zk3/KJnd29c93jA33H/et4dpI/qKonJ/nbJJ9Jsnejz53WmPxykrdV1Sfy/7uiOyW5S5LTD/amSeE7k8X8sACAxVr/b/lB7Ely7LrHx2RtHur6c1yV5LFJUlW3SPK47t7wf00bNibd/eaqumvWxpGOzlp3tCfJBd193UbvBQAOjzrysAY017sgyQlVdXzWkpDTkjzxO+qquk2Sz3f3viTPSXLWtJNOS0wyOdl7D6ViAGA1dffeqjo9yVuSbE9yVndfVFXPS7K7u3cleUiS36mqztpQztOnnXdqYwIALLdtRwySmKS7z01y7n7Pnbnu/jlJzpnlnDZYAwCWhsQEAEaujlydnEFjAgAjN9RQziKsTosFAIyexAQARm6g5cILITEBAJaGxAQARm6V5phoTABg5AzlAAAsgMQEAEZulYZyJCYAwNKQmADAyNX21UlMNCYAMHLbVqgxMZQDACwNiQkAjFxtk5gAAMydxAQARq62r07OsDrfBAAYPYkJAIzcKq3KWXhjcvV5ly36I+bqdo+429AlzOyMM88fuoSZ3WVkP+ZzX/muoUuY2UkP/YGhS5jZrW5zy6FLmMknr/ja0CXM7L23u/fQJczkwXe4dOgSDtHdD+unmfwKALAAhnIAYORWaShHYgIALA2JCQCMnGvlAABLo7atzgDI6nwTAGD0JCYAMHKWCwMALIDEBABGbpWWC2tMAGDkDOUAACyAxAQARs5yYQCABZCYAMDIrdIcE40JAIzcKq3KMZQDACwNiQkAjNwqDeVITACApSExAYCRs1wYAGABJCYAMHKrNMdEYwIAI7dKjckhD+VU1VPmWQgAwL9mjslzD/ZCVe2oqt1Vtfudb9z5r/gIAGCa2lZzvw1lw6GcqvrIwV5KcvuDva+7dybZmSQve0f6kKsDALaUaXNMbp/kx5J8Yb/nK8nfLaQiAGAmq7RceFpj8sYkt+juC/d/oareuZCKAICZrNK1cjZsTLr7qRu89sT5lwMAbGWWCwPAyFkuDACwABITABi5rTT5FQBYcoZyAAAWQGICACMnMQEAWACJCQCM3CpNfl2dbwIAjJ7EBABGbpXmmGhMAGDkDOUAACyAxAQAxq5WZyhHYgIALA2JCQCMnMmvAMDSMPkVAGABJCYAMHKrNJQjMQEAlsbCE5NH3e59i/6IuTrjzPOHLmFmj33eg4YuYWYP2L1z6BJm8uTnHzV0CTO7Mj10CTO7xRHfHLqEmXzgqu8ZuoSZfeiSa4cuYSav2H3k0CUcktf//uH9vFWaY2IoBwBGzlAOAMACSEwAYOQkJgAACyAxAYCxW6HJr6vzTQCA0ZOYAMDI1QpdXVhjAgAjt0r7mKzONwEARk9iAgAjZ7kwALDlVdWpVXVZVV1eVWcc4PU7VdU7qupDVfWRqvrxaeeUmADA2A0wx6Sqtid5cZKHJ9mT5IKq2tXdF6877L8m+fPu/sOqukeSc5Mct9F5NSYAMHIDDeWckuTy7r4iSarq7CSPSbK+Mekk3zW5f1SSq6adVGMCAByKo5Ncue7xniT33++Y30pyXlU9I8nNk/zotJOaYwIAI1e1bQG32lFVu9fdduz/sQcopfd7/IQkf9rdxyT58ST/t6o27D0kJgDADXT3ziQ7NzhkT5Jj1z0+JjccqnlqklMn53tPVd0kyW2SXH2wk0pMAGDsttX8b9NdkOSEqjq+qm6U5LQku/Y75tNJHpYkVXX3JDdJcs1GJ5WYAMDIDbHza3fvrarTk7wlyfYkZ3X3RVX1vCS7u3tXkv+c5CVV9StZG+Z5cnfvP9zzHTQmAMAh6e5zs7YEeP1zZ667f3GSB85yTo0JAIycnV8BABZgamNSVSdW1cOq6hb7PX/q4soCADatts3/NpANP7mqnpnkL5M8I8nHquox617+74ssDADYeqbNMXlakvt191er6rgk51TVcd39ohx4YxUA4DDbSnNMtnf3V5Okuz+V5CFJHllVL8gGjcn63eJe8dq/mFetAMCBbNs2/9tApiUmn62qk7r7wiSZJCePSnJWknsd7E3rd4u75qL3bbheGQDgetMak59Psnf9E929N8nPV9UfL6wqAGDTqlZnKGfDxqS792zw2rvnXw4AsJXZYA0Axm7AOSHzpjEBgJHbSqtyAAAOG4kJAIzdgDu1ztvqfBMAYPQkJgAwdis0x0RjAgAjV4ZyAADmT2ICAGO3QkM5EhMAYGlITABg5GqFdn5dnW8CAIyexAQAxm6rXF0YABgBQzkAAPMnMQGAsVuhoRyJCQCwNCQmADByq7RcWGMCAGO3QtfKWXhjcv7XTl70R8zVXe42dAWze8DunUOXMLO/O3nH0CXM5L4ffc3QJczs2Jf80tAlzOzKp/3h0CXM5C63/fLQJczskk/ceOgSZnLyA48fugQOM4kJAIyda+UAAMyfxAQARq7MMQEAloahHACA+ZOYAMDYrdBQzup8EwBg9CQmADB2rpUDADB/EhMAGDvXygEAlobJrwAA8ycxAYCxs8EaAMD8SUwAYOxWaI6JxgQAxs4+JgAA8ycxAYCxW6F9TFbnmwAAoycxAYCxW6E5JhoTABi7rbQqp6pOSdLdfUFV3SPJqUku7e5zF14dALClbNiYVNVvJnlkkiOq6q+T3D/JO5OcUVX36e7/tvgSAYANrdDk12mJyU8lOSnJjZN8Nskx3f3lqvqfSd6XRGMCAMzNtBZrb3df191fT/LJ7v5yknT3N5LsO9ibqmpHVe2uqt3nveElcywXALiBqvnfBjItMflWVd1s0pjc7/onq+qobNCYdPfOJDuT5A3vv67nUSgAsPqmNSYP7u5/SZLuXt+IHJnkSQurCgDYvK2yKuf6puQAz/9Tkn9aSEUAwGxWaB+T1WmxAIDRs8EaAIzdCi0XXp1vAgCMnsQEAEauV2iOicYEAMZuhVblrM43AQBGT2ICAGMnMQEAmD+JCQCMnMmvAMDyMJQDADB/EhMAGLsVGsqRmAAAS0NiAgBjt0LXytGYAMDIrdKqnNVpsQCA0ZOYAMDYWS4MADB/EhMAGLmWmAAAzN/CE5PPfWH7oj9irs595buGLmFmT37+UUOXMLP7fvQ1Q5cwkw/e6/FDlzCz4y5559AlzOyyq8b1e/nE239x6BJm9pWv/MvQJczk3W/6wNAlHJJff/wDDu8HWpUDACyLrm1zv21GVZ1aVZdV1eVVdcYBXn9hVV04uX28qqZ28+aYAAAzq6rtSV6c5OFJ9iS5oKp2dffF1x/T3b+y7vhnJLnPtPNKTABg7Krmf5vulCSXd/cV3f2tJGcnecwGxz8hyaunnVRjAgAciqOTXLnu8Z7JczdQVd+X5Pgkb592UkM5ADB2C1guXFU7kuxY99TO7t65/pADvK0PcrrTkpzT3ddN+1yNCQCM3CKulTNpQnZucMieJMeue3xMkqsOcuxpSZ6+mc81lAMAHIoLkpxQVcdX1Y2y1nzs2v+gqrpbku9O8p7NnFRiAgBjN8DOr929t6pOT/KWJNuTnNXdF1XV85Ls7u7rm5QnJDm7uw82zPMdNCYAwCHp7nOTnLvfc2fu9/i3ZjmnxgQARq4POA91nDQmADByLuIHALAAEhMAGDuJCQDA/ElMAGDkFrHB2lAkJgDA0pCYAMDIrdKqHI0JAIydoRwAgPmTmADAyK3SUM7qfBMAYPRmbkyq6hWLKAQAODSdmvttKBsO5VTVrv2fSvLQqrpVknT3oxdVGACwOVtpKOeYJF9O8oIk/2ty+8q6+wdUVTuqandV7X7XuTvnVSsAsOKmTX49OcmzkvxGkl/t7gur6hvd/Tcbvam7dybZmSR/9Jb0XCoFAA5shZYLb9iYdPe+JC+sqtdOfv3ctPcAAByqTTUZ3b0nyU9X1U9kbWgHAFgSvUKLbGdKP7r7TUnetKBaAIBD4CJ+AAALYL4IAIzcVlouDABw2EhMAGDkhtypdd4kJgDA0pCYAMDIrdIcE40JAIyc5cIAAAsgMQGAkTP5FQBgASQmADByJr8CAEvDUA4AwAJITABg5FZpKGd1vgkAMHoSEwAYuVWaY6IxAYCRM5QDALAAEhMAGDlDOTP42EVfWvRHzNVJD/2BoUuY2ZXpoUuY2bEv+aWhS5jJcZe8c+gSZvapuz9k6BJm9vnXXzp0CTO54x32DF3CzD5zxU2GLmEmJ55y4tAlcJhJTABg5FxdGABgASQmADBy3auTmGhMAGDkeoUGQFbnmwAAoycxAYCRW6XlwhITAGBpSEwAYORWKTHRmADAyK1SY2IoBwBYGhITABg5iQkAwAJITABg5Oz8CgAsDUM5AAALIDEBgJGTmAAALIDEBABGTmICALAAEhMAGDnLhQGApbFvhYZyZmpMqupBSU5J8rHuPm8xJQEAW9WGc0yq6v3r7j8tyR8kuWWS36yqMxZcGwCwCZ2a+20o0ya/Hrnu/o4kD+/u5yZ5RJL/sLCqAIAtaVpjsq2qvruqbp2kuvuaJOnuryXZe7A3VdWOqtpdVbsves+fzq9aAOAGumvut6FMm2NyVJIPJKkkXVV36O7PVtUtJs8dUHfvTLIzSU5/wZd6XsUCADe0SvuYbNiYdPdxB3lpX5KfnHs1AMCWdkjLhbv760n+fs61AACHYJX2MbHzKwCwNGywBgAjt2XmmAAAy89QDgDAAkhMAGDk9g1dwBxJTACApSExAYCRM8cEAGABJCYAMHKWCwMAS8NQDgDAAmhMAGDkOjX322ZU1alVdVlVXV5VZxzkmJ+pqour6qKqetW0cxrKAQBmVlXbk7w4ycOT7ElyQVXt6u6L1x1zQpLnJHlgd3+hqm437bwaEwAYuX09yMeekuTy7r4iSarq7CSPSXLxumOeluTF3f2FJOnuq6ed1FAOAIzcQEM5Rye5ct3jPZPn1rtrkrtW1bur6r1Vdeq0k0pMAIAbqKodSXase2pnd+9cf8gB3rZ/dnNEkhOSPCTJMUneVVX37O4vHuxzNSYAMHKLWC48aUJ2bnDIniTHrnt8TJKrDnDMe7v72iR/X1WXZa1RueBgJzWUAwAciguSnFBVx1fVjZKclmTXfsf8RZKHJklV3SZrQztXbHRSiQkAjFwPMPm1u/dW1elJ3pJke5Kzuvuiqnpekt3dvWvy2iOq6uIk1yX51e7+543OW73gb/P4Z//DMHOFD9GtbnPLoUuY2bN+6ptDlzCzr+69+dAlzOSyq48auoSZff5Lo/qjlyS5y2NPHLqEmbzq2W8fuoSZ3erW4/qzd/lHPzV0CYfkLS8/6bBuxfq2j35z7n/gH3avmwyynayhHABgaRjKAYCRc60cAIAFkJgAwMgNMfl1USQmAMDSkJgAwMht9mrAY6AxAYCRG+gifgthKAcAWBoSEwAYOcuFAQAWQGICACO3SsuFNSYAMHL7VmhVjqEcAGBpSEwAYORWaShHYgIALA2JCQCM3CotF9aYAMDI2fkVAGABJCYAMHImvwIALIDEBABGrrfKBmtVdf+q+q7J/ZtW1XOr6q+q6vlVddThKREA2CqmDeWcleTrk/svSnJUkudPnnvZAusCADZpX8//NpRpQznbunvv5P7J3X3fyf3zq+rCBdYFAGzSVpr8+rGqesrk/oer6uQkqaq7Jrn2YG+qqh1Vtbuqdn/yI6+aU6kAwKqb1pj8QpIfrqpPJrlHkvdU1RVJXjJ57YC6e2d3n9zdJ9/53k+cX7UAwA10z/82lA2Hcrr7S0meXFW3TPL9k+P3dPfnDkdxAMDWsqnlwt39lSQfXnAtAMAh2OdaOQDAsthKk18BAA4biQkAjJzEBABgASQmADByQ+7UOm8aEwAYuV6hVTmGcgCApSExAYCRM/kVAGABJCYAMHImvwIAS8NQDgDAAkhMAGDkJCYAAAsgMQGAkVulya8SEwBgaUhMAGDkVmmOicYEAEZu376hK5gfQzkAwNKQmADAyBnKmcGDHnqnRX/EXH3yiq8NXcLMPnDV9wxdwszuctsvD13CTE68/ReHLmFmd7zDnqFLmNmvPfvtQ5cwkyf+3o8MXcLMXn/m+UOXMJN73v+EoUvgMJOYAMDISUwAgKVhHxMAgAWQmADAyPVCxnJqAeecTmICACwNiQkAjJzJrwDA0rDzKwDAAkhMAGDkVmkoR2ICACwNiQkAjJwN1gAAFkBiAgAjt0pzTDQmADByvZCxHDu/AgBbnMQEAEbO5FcAgAWQmADAyJn8CgAsjX0rNJZjKAcAWBoSEwAYuVUaytkwMamqZ1bVsYerGABga5s2lPPbSd5XVe+qqv9YVbc9HEUBAJvXPf/bUKY1JlckOSZrDcr9klxcVW+uqidV1S0P9qaq2lFVu6tq97vfvHOO5QIA+9vXPffbUKbNMenu3pfkvCTnVdWRSR6Z5AlJfi/JAROU7t6ZZGeS/J83rdLIFwCwSNMak+/YKL+7r02yK8muqrrpwqoCADat9w1dwfxMG8p5/MFe6O5vzLkWAGCL2zAx6e6PH65CAIBD0ys0a8IGawDA0tCYAMDI7ds3/9tmVNWpVXVZVV1eVWcc4PUnV9U1VXXh5PYL085p51cAGLkhhnKqanuSFyd5eJI9SS6oql3dffF+h76mu0/f7HklJgDAoTglyeXdfUV3fyvJ2Uke8689qcYEAEZuX8//tglHJ7ly3eM9k+f297iq+khVnbOZy9xoTACAG1i/i/vktmP/Qw7wtv1bmr9Kclx33zvJW5O8fNrnmmMCACPXm4w4Zjrnul3cD2JPkvUJyDFJrtrvHP+87uFLkjx/2udKTABg5Aa6iN8FSU6oquOr6kZJTsva7vDfVlV3XPfw0UkumXZSiQkAMLPu3ltVpyd5S5LtSc7q7ouq6nlJdnf3riTPrKpHJ9mb5PNJnjztvBoTABi5fQsYytmM7j43ybn7PXfmuvvPSfKcWc5pKAcAWBoSEwAYuVW6Vo7GBABGrje5hfwYGMoBAJaGxAQARm7fCg3lSEwAgKUhMQGAkVulya8SEwBgaUhMAGDkhtpgbREW3pj89J0vXPRHzNV7b3fvoUuY2YcuuXboEmZ2ySduPHQJM/nKV/5l6BJm9pkrbjJ0CTO7451uPnQJM3n9mecPXcLMHvu8Bw1dwkxutPsjQ5cwCis0kmMoBwBYHoZyAGDkeoWGciQmAMDSkJgAwMit0gZrGhMAGDlDOQAACyAxAYCRk5gAACyAxAQARm6FAhONCQCMnaEcAIAFkJgAwMj1Cu1jIjEBAJaGxAQARm6fOSYAAPMnMQGAkVulOSYaEwAYOcuFAQAWQGICACMnMQEAWACJCQCM3L6tMvm1qm6U5LQkV3X3W6vqiUkekOSSJDu7+9rDUCMAsIFVGsqZlpi8bHLMzarqSUlukeT1SR6W5JQkT1pseQDAVjKtMblXd9+7qo5I8pkk39vd11XVnyX58MHeVFU7kuxIkt997m/k537mcXMrGAD4TltpH5Ntk+Gcmye5WZKjknw+yY2THHmwN3X3ziQ7k+Szl35odX5aAMBCTWtMXprk0iTbk/xGktdW1RVJfjDJ2QuuDQDYhFW6Vs6GjUl3v7CqXjO5f1VVvSLJjyZ5SXe//3AUCABsbCtNfk13X7Xu/heTnLPQigCALcs+JgAwcqs0+dXOrwDA0pCYAMDI9b59Q5cwNxITAGBpSEwAYOS2zHJhAGD5mfwKALAAEhMAGLlV2mBNYgIALA2JCQCM3ColJhoTABi5fW0fEwCAuZOYAMDIrdJQjsQEAFgaEhMAGLlVSkw0JgAwcnZ+BQBYAIkJAIzcvn2WCwMAzN3CE5O3XnPSoj9irh58h0uHLmFmr9h95NAlzOzkBx4/dAkzefebPjB0CTM78ZQThy5hZpd/9FNDlzCTe97/hKFLmNmNdn9k6BJm8q2T7z10CYfm2ssO68et0uRXiQkAsDTMMQGAkesV2pJeYwIAI2coBwBgASQmADByEhMAgAWQmADAyO0z+RUAWBaGcgAAFkBiAgAj166VAwAwfxITABi5VZpjojEBgJFbpS3pDeUAAEtDYgIAI7dvhYZyJCYAwNKQmADAyFkuDACwABITABg5y4UBgKVhuTAAsOVV1alVdVlVXV5VZ2xw3E9VVVfVydPOOTUxqao7J/nJJMcm2ZvkE0le3d1fmqF2AGBBhhjKqartSV6c5OFJ9iS5oKp2dffF+x13yyTPTPK+zZx3w8Skqp6Z5I+S3CTJv01y06w1KO+pqofM+B0AgNVxSpLLu/uK7v5WkrOTPOYAx/12kt9N8s3NnHRaYvK0JCd193VV9YIk53b3Q6rqj5P8ZZL7bLp8AGAhBloufHSSK9c93pPk/usPqKr7JDm2u99YVc/ezEk3M/n1iCTXJblxklsmSXd/uqqOPNgbqmpHkh2Th7/Y3Ts3U8ysqmrH/M999/mebp3F1Ju8/vfnfcb/b1E1L8qi6v31xz9g3qf8trH9jJNF1vw98z9l/IwPl4XUfO1lcz3demP8GR/M+X/1wzXvc+73b3mS7Nzv53Wgz/z2mFJVbUvywiRPnulzuw8+LlVVz0ry1CTvTfLgJM/v7pdV1W2TvK67HzzLh81bVe3u7qkTaZbF2OpNxlfz2OpN1Hw4jK3eRM2Hw9jqXTZV9UNJfqu7f2zy+DlJ0t2/M3l8VJJPJvnq5C13SPL5JI/u7t0HO++GiUl3v6iq3pq1GOEF3X3p5PlrstaoAABb0wVJTqiq45N8JslpSZ54/YuTRTK3uf5xVb0zybM3akqSTQzldPdFSS46tJoBgFXU3Xur6vQkb0myPclZ3X1RVT0vye7u3nUo5x37BmtjGxscW73J+GoeW72Jmg+HsdWbqPlwGFu9S6e7z01y7n7PnXmQYx+ymXNuOMcEAOBwsvMrALA0RtmYbHYL3GVRVWdV1dVV9bGha9mMqjq2qt5RVZdU1UWT1VlLrapuUlXvr6oPT2p+7tA1bUZVba+qD1XVG4euZTOq6lNV9dGqurCqNpzAtiyq6lZVdU5VXTr5Pf1DQ9e0kaq62+Tne/3ty1X1y0PXtZGq+pXJn7uPVdWrq+omQ9c0TVU9a1LvRcv+891qRjeUM9kC9+NZtwVukifsvwXuMqmqB2dtudQruvueQ9czTVXdMckdu/uDk62EP5Dk3y/5z7iS3Ly7vzrZY+f8JM/q7vcOXNqGquo/JTk5yXd196OGrmeaqvpUkpO7+5+GrmWzqurlSd7V3X9SVTdKcrPu/uLQdW3G5O+7zyS5f3f/w9D1HEhVHZ21P2/36O5vVNWfZ20zzj8dtrKDq6p7Zm2X0lOSfCvJm5P8Und/YtDCSDLOxGSzW+Auje7+26yt3R6F7v7H7v7g5P5XklyStR3+llavuX6t/JGT21J33VV1TJKfSPInQ9eyqqrqu7K2tcFLk6S7vzWWpmTiYUk+uaxNyTpHJLlpVR2R5GZJrhq4nmnunuS93f317t6b5G+ydk04lsAYG5MDbYG71P9ojllVHZe1Sw9s6uJLQ5oMi1yY5Ookf93dy17z/07yX5KM6XrlneS8qvrAZFfIZff9Sa5J8rLJkNmfVNXNhy5qBqclefXQRWykuz+T5PeSfDrJPyb5UnefN2xVU30syYOr6tZVdbMkP56168CxBMbYmGy4BS7zU1W3SPK6JL/c3V8eup5puvu67j4pyTFJTpnEtUupqh6V5Oru/sDQtczogd193ySPTPL0yTDlMjsiyX2T/GF33yfJ15Is/by0JJkMOz06yWuHrmUjVfXdWUutj0/yvUluXlU/O2xVG+vuS5I8P8lfZ20Y58NJ9g5aFN82xsZkT76zsz0myx8bjs5knsbrkryyu18/dD2zmET170xy6sClbOSBSR49mbNxdpIfqao/G7ak6br7qsmvVyd5Q9aGVpfZniR71qVn52StURmDRyb5YHd/buhCpvjRJH/f3dd097VJXp9kcReXmpPufml333dyaZXPJzG/ZEmMsTH59ha4k/9RnJbkkHaX48AmE0lfmuSS7n7B0PVsRlXdtqpuNbl/06z9ZXnpsFUdXHc/p7uP6e7jsvZ7+O3dvdT/y6yqm08mQ2cyHPKIrEXiS6u7P5vkyqq62+SphyVZ2knc+3lClnwYZ+LTSX6wqm42+bvjYVmbl7bUqup2k1/vlOSxGcfPeksY3c6vB9sCd+CyNlRVr07ykCS3qao9SX6zu186bFUbemCSn0vy0cmcjST59ckOf8vqjklePlnFsC3Jn3f3KJbgjsjtk7xh7d+eHJHkVd395mFL2pRnJHnl5D8yVyR5ysD1TDWZ9/DwJL84dC3TdPf7qvxjX5QAAABVSURBVOqcJB/M2nDIhzKOHVVfV1W3TnJtkqd39xeGLog1o1suDACsrjEO5QAAK0pjAgAsDY0JALA0NCYAwNLQmAAAS0NjAgAsDY0JALA0NCYAwNL4f/XIoG0hM8+kAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "models_dir = \"./checkpoint/figure_model/\"\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "total_task = 5\n",
    "tasknum = 4\n",
    "\n",
    "model_dir = os.path.join(models_dir, 'joint_base_{}_{}_tri{}_{}_{}_{}.pt'.format(args.trainer, args.dataset,args.CCtriplet, total_task, tasknum, args.model))\n",
    "saved_model = torch.load(model_dir, map_location='cuda:0')\n",
    "\n",
    "numclass=10\n",
    "\n",
    "net = resnet32(num_classes=numclass, tasknum=total_task)\n",
    "\n",
    "net.load_state_dict(saved_model)\n",
    "\n",
    "net.eval()\n",
    "net.cuda()\n",
    "\n",
    "global feature_output_list\n",
    "feature_output_list = []\n",
    "mtx = get_avg_feature_map(net)\n",
    "\n",
    "mp.subplots(figsize=(10,8))\n",
    "sns.heatmap(mtx, annot=False, annot_kws={'size':20}, cmap=\"coolwarm\", vmin=0.4, vmax=0.9)\n",
    "\n",
    "mp.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch_project",
   "language": "python",
   "name": "pytorch_project"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}