Spam Ham Email Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import math\n",
    "import copy\n",
    "import itertools\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read and store dataset functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Document class to store email instances\n",
    "class Document:\n",
    "    text = \"\"\n",
    "    word_freqs = {}\n",
    "    actual_label = \"\"\n",
    "    predicted_label = \"\"\n",
    "\n",
    "    def __init__(self, text, counter, actual_label):\n",
    "        self.text = text\n",
    "        self.word_freqs = counter\n",
    "        self.actual_label = actual_label\n",
    "\n",
    "    def getText(self):\n",
    "        return self.text\n",
    "\n",
    "    def getWordFreqs(self):\n",
    "        return self.word_freqs\n",
    "\n",
    "    def getActualLabel(self):\n",
    "        return self.actual_label\n",
    "\n",
    "    def getPredictedLabel(self):\n",
    "        return self.predicted_label\n",
    "\n",
    "    def setPredictedLabel(self, guess_label):\n",
    "        self.predicted_label = guess_label\n",
    "\n",
    "# extract words with their frequency in text\n",
    "def bagOfWords(text):\n",
    "    bagsofwords = collections.Counter(re.findall(r'\\w+', text))\n",
    "    return dict(bagsofwords)\n",
    "\n",
    "# Read all text files in the given directory and construct the data set D\n",
    "def makeDataSet(storage_dict, directory, actual_label):\n",
    "    for dir_entry in os.listdir(directory):\n",
    "        dir_entry_path = os.path.join(directory, dir_entry)\n",
    "        if os.path.isfile(dir_entry_path):\n",
    "            with open(dir_entry_path, 'r', encoding=\"ansi\", errors='ignore') as text_file:\n",
    "                text = text_file.read()\n",
    "                storage_dict.update({dir_entry_path: Document(text, bagOfWords(text), actual_label)})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and Test Multinomial Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracts the vocabulary of all the text in a data set\n",
    "def extractVocab(data_set):\n",
    "    all_text = \"\"\n",
    "    v = []\n",
    "    for x in data_set:\n",
    "        all_text += data_set[x].getText()\n",
    "    for y in bagOfWords(all_text):\n",
    "        v.append(y)\n",
    "    return v\n",
    "\n",
    "# Training\n",
    "def trainMultinomialNaiveBayes(training, priors, cond):\n",
    "    v = extractVocab(training)\n",
    "    n = len(training)\n",
    "    for c in classes:\n",
    "        n_c = 0.0\n",
    "        text_c = \"\"\n",
    "        for i in training:\n",
    "            if training[i].getActualLabel() == c:\n",
    "                n_c += 1\n",
    "                text_c += training[i].getText()\n",
    "        priors[c] = float(n_c) / float(n)\n",
    "        token_freqs = bagOfWords(text_c)\n",
    "        # Calculate conditional probabilities for each token and sum using laplace smoothing and log-scale\n",
    "        for t in v:\n",
    "            # known word case\n",
    "            if t in token_freqs:\n",
    "                cond.update({t + \"_\" + c: (float((token_freqs[t] + 1.0)) / float((len(text_c) + len(token_freqs))))})\n",
    "            # unknown word case\n",
    "            else:\n",
    "                cond.update({t + \"_\" + c: (float(1.0) / float((len(text_c) + len(token_freqs))))})\n",
    "\n",
    "\n",
    "# Testing. \n",
    "# Returns classification guess\n",
    "def applyMultinomialNaiveBayes(data_instance, priors, cond):\n",
    "    score = {}\n",
    "    for c in classes:\n",
    "        score[c] = math.log10(float(priors[c]))\n",
    "        for t in data_instance.getWordFreqs():\n",
    "            if (t + \"_\" + c) in cond:\n",
    "                score[c] += float(math.log10(cond[t + \"_\" + c]))\n",
    "    if score[\"spam\"] > score[\"ham\"]:\n",
    "        return \"spam\"\n",
    "    else:\n",
    "        return \"ham\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Evaluation of Model - Confusion Matrix Plot\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix'):\n",
    "   \n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "    \"\"\"\n",
    "    plt.figure()\n",
    "    cmap=plt.cm.Blues\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stores emails as dictionaries. file_name : email_Document\n",
    "training_set = dict()\n",
    "test_set = dict()\n",
    "# Define classes for dataset\n",
    "# ham = 0 for not spam, spam = 1 for is spam\n",
    "classes = [\"ham\", \"spam\"]\n",
    "\n",
    "# takes directories holding the data text files as paramters\n",
    "def makeDatasets(training_spam_dir, training_ham_dir, test_spam_dir, test_ham_dir):\n",
    "    makeDataSet(training_set, training_spam_dir, classes[1])\n",
    "    makeDataSet(training_set, training_ham_dir, classes[0])\n",
    "    makeDataSet(test_set, test_spam_dir, classes[1])\n",
    "    makeDataSet(test_set, test_ham_dir, classes[0])\n",
    "\n",
    "makeDatasets('train/spam', 'train/ham', 'test/spam', 'test/ham')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and Test on dataset with stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============================CLASSIFICATION PERFORMANCE REPORT WITHOUT STOPWORDS REMOVED===============================\n",
      "No. of true positives: 1467\n",
      "No. of true negatives: 3382\n",
      "No. of false positives: 290\n",
      "No. of false negatives: 33\n",
      "Total documents in test set: 5172\n",
      "Accuracy: 93.75483372003094\n",
      "Precision 83.49459305634605\n",
      "Recall: 97.8\n",
      "F1-score: 90.08289837273564\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAEYCAYAAAA+mm/EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5wV1f3G8c+zSxGlKiBFUFSKLWJDo4ldxBJBY8HYC6hREzXNGFvsJpbYo8SexJIoViJi+RlN7ARULDQLCCrYMYiU7++PmSXX5TZgL/fe3efta15775kzM2dY+XLazFFEYGZmS6opdwHMzCqVA6SZWQ4OkGZmOThAmpnl4ABpZpaDA6SZWQ4OkIakVpIelPS5pL8tx3kOkvRoQ5atXCR9X9Jb5S6HlZc8D7J6SPoRcArQD/gSGAecHxHPLOd5DwFOBLaOiAXLXdAKJymA3hExudxlscrmGmSVkHQK8AfgAmB1oCdwLTC4AU6/JjCxKQTHYkhqVu4yWIWICG8VvgHtgDnAfnnytCQJoDPS7Q9Ay3Tf9sB04GfAR8BM4Ih032+Bb4D56TWOAs4G/pxx7rWAAJql3w8HppLUYt8GDspIfybjuK2BF4HP059bZ+z7P+Bc4F/peR4FOua4t7ry/zKj/EOA3YGJwCfAaRn5BwDPAp+lea8GWqT7/pney1fp/R6Qcf5fAR8At9elpcesk15j0/R7N2A2sH25/9/wVtrNNcjq8F1gJWBknjy/AbYC+gMbkwSJ0zP2dyEJtN1JguA1kjpExFkktdK7IqJ1RNyYryCSVgGuBHaLiDYkQXBclnyrAg+neVcDLgMelrRaRrYfAUcAnYEWwM/zXLoLyZ9Bd+BMYARwMLAZ8H3gTElrp3kXAicDHUn+7HYCfgwQEdumeTZO7/eujPOvSlKbHp554YiYQhI8/yJpZeBm4JaI+L885bVGwAGyOqwGzI78TeCDgHMi4qOImEVSMzwkY//8dP/8iBhFUnvqu4zlWQRsKKlVRMyMiAlZ8uwBTIqI2yNiQUTcAbwJ/CAjz80RMTEi5gJ3kwT3XOaT9LfOB+4kCX5XRMSX6fUnAN8BiIiXI+K59LrvANcD2xVxT2dFxLy0PN8SESOAScDzQFeSf5CskXOArA4fAx0L9I11A97N+P5umrb4HPUC7H+B1ktbkIj4iqRZeiwwU9LDkvoVUZ66MnXP+P7BUpTn44hYmH6uC2AfZuyfW3e8pD6SHpL0gaQvSGrIHfOcG2BWRHxdIM8IYEPgqoiYVyCvNQIOkNXhWeBrkn63XGaQNA/r9EzTlsVXwMoZ37tk7oyI0RGxC0lN6k2SwFGoPHVlen8Zy7Q0riMpV++IaAucBqjAMXmnc0hqTdKveyNwdtqFYI2cA2QViIjPSfrdrpE0RNLKkppL2k3S79JsdwCnS+okqWOa/8/LeMlxwLaSekpqB/y6boek1SXtlfZFziNpqi/Mco5RQB9JP5LUTNIBwPrAQ8tYpqXRBvgCmJPWbo+rt/9DYO0ljsrvCuDliDiapG/1j8tdSqt4DpBVIiIuI5kDeTowC5gGnADcl2Y5D3gJeAV4FRibpi3LtcYAd6XneplvB7UaktHwGSQju9uRDoDUO8fHwJ5p3o9JRqD3jIjZy1KmpfRzkgGgL0lqt3fV2382cKukzyTtX+hkkgYDg0i6FSD5PWwq6aAGK7FVJE8UNzPLwTVIM7McHCDNzHJwgDQzy8EB0swsh0b9UL6atQq1aFPuYlg9G/XtUe4iWBavjBs7OyI6NdT5atuuGbFgiYeSlhBzZ42OiEENdd2G1LgDZIs2tOxbcBaHrWCPPnV5uYtgWXRp16L+k0/LJRbMLerv39fjrin0lFPZNOoAaWZlJEFNbblLsVwcIM2sdFTdwxwOkGZWOir0CHxlc4A0sxKRa5BmZlkJ90GamWUnN7HNzHJyE9vMLAfXIM3MsvA8SDOzPNzENjPLxtN8zMxyq3EfpJnZkjwP0swsFzexzcxy8zQfM7McXIM0M8vC8yDNzPKo8iZ2ddd/zayCpYM0hbZCZ5FWkvSCpPGSJkj6bZreS9LzkiZJuktSizS9Zfp9crp/rYxz/TpNf0vSroWu7QBpZqUjFd4KmwfsGBEbA/2BQZK2Ai4GLo+I3sCnwFFp/qOATyNiXeDyNB+S1geGAhsAg4BrJeXtA3CANLPSkKCmWeGtgEjMSb82T7cAdgT+nqbfCgxJPw9Ov5Pu30mS0vQ7I2JeRLwNTAYG5Lu2A6SZlU7D1CCRVCtpHPARMAaYAnwWEQvSLNOB7unn7sA0gHT/58BqmelZjsnKgzRmVjrFTfPpKOmljO83RMQNmRkiYiHQX1J7YCSwXpbzRN1Vc+zLlZ6TA6SZlUbx03xmR8TmxWSMiM8k/R+wFdBeUrO0lrgGMCPNNh3oAUyX1AxoB3ySkV4n85is3MQ2s9JpgCa2pE5pzRFJrYCdgTeAJ4F902yHAfennx9Iv5PufyIiIk0fmo5y9wJ6Ay/ku7ZrkGZWMmqYeZBdgVvTEeca4O6IeEjS68Cdks4D/gPcmOa/Ebhd0mSSmuNQgIiYIOlu4HVgAXB82nTPyQHSzEpCNEyAjIhXgE2ypE8lyyh0RHwN7JfjXOcD5xd7bQdIMysNCfl9kGZm2TVQE7tsHCDNrGQcIM3McnCANDPLQu6DNDPLzTVIM7McHCDNzHJwgDQzy0a4D9LMLBsh1yDNzHJxgDQzy6W646MDpJmViKCmprrfqOgAaWYl4ya2mVkWHqSxZdayRTMeu/EkWrRoRrPaWkY+9h/O++MorjvrR2y6fk+EmPzeRww783a+mvsNPbp0YMQ5h9CuTStqa2o446r7Gf3M6+y4ZT/O/cletGjejG/mL+C0P9zHUy9OLPftNQrvT5/GicceyawPP0A1NRxy+NEMO+5EJrw6nl+efAJffTWHHj3X5NoRt9GmbVsArrz0Yv56+y3U1tZw3sWXs8POA8t8F2VW3fHRAbJc5n2zgEHDr+Srud/QrFkNT9x0Co/+63V+ecm9fPnV1wBc/LN9OG7odlxy8xh+dfQg7hkzlhF/e4Z+a3fhvquOo98eZ/HxZ3PY96TrmTnrc9ZfpysPXns86+x6epnvrnFo1qwZZ5/3O77TfxPmfPklA7fbkm132IlTTjyWs867mK2/ty1/vf0Wrr3yUn51+m95683Xue/eu3nq+XF8MHMG+w/ejX+PnUBtbVHrsjQ+jaAPsrpLX+W+mvsNAM2b1dKsWS0RsTg4AqzUsjnJUhoQEbRdZSUA2rVuxcxZnwMw/q3piz+/PmUmLVs0p0Vz/7vXEFbv0pXv9E9eZN26TRt69+3HBzNmMGXyRL67zfcB2G6HnXjogZEAjH74QYbssz8tW7ZkzbV60WvtdfjPyy+WrfyVQFLBrZI5QJZRTY147s5Tee/xi3jiuTd58bV3Abj+7IN557EL6LvW6lx751MAnH/9KIbuPoDJj5zLyKuO45SL/7bE+fbeuT/j35rGN/MXLLHPls97777Da6+MZ9PNB9BvvQ0YPepBAB687x5mvD8dgJkzZ9BtjTUWH9O1W3dmzni/LOWtGCpiq2ArNEBKWkvSayvympVs0aJgq6EXse6up7P5hmuy/jpdATjm7D+z9sDf8ObbH7DvwM0A2H/Q5vz5wedYd9AZ7H3iddx43qHf+td3vbW7cN5PBnPCeXeW5V4as6/mzOHoQw7gnAsvoU3btlx+zQ3cPOKPDNx2S+bM+ZIWzVsALK7tZ6r0GlIpSaKmpqbgVskqu3RNxOdz5vLPlyYxcOv1F6ctWhT8/dGxDNmpPwCHDfku9zw6FoDnX3mblVo0p2P7VQDo3rk9d102nKPPuJ23p89e8TfQiM2fP5+jDjmAffY/kD322huA3n36cdd9o3j0n8+z974HsGavtQHo1q07M6ZPX3zszBnv06Vrt7KUu1K4ib30aiWNkDRB0qOSWkkaJulFSeMl3SNpZQBJt0i6TtKTkqZK2k7STZLekHRLGcreYDp2aE271q2ApK9xxy37MvHdD1m7R8fFefbYdiMmvvMhANM++ITtB/QFoG+v1VmpZXNmfTqHdq1bce9Vx3LmVQ/w7PipK/5GGrGI4OQThtO7bz+OPeGkxemzZn0EwKJFi7j89xdy6JHDARi4+57cd+/dzJs3j3ffeZupUyazyWZblKXslaLaA2Q5evN7AwdGxLB0jdofAvdGxAiAdI3bo4Cr0vwdgB2BvYAHgW2Ao4EXJfWPiHGZJ5c0HEj+j23euvR3s4y6dGzLiHMOobamhpoacc+Ysfzj6Qk8ftNJtFmlFRK8OvF9fnLBXQCcetlIrj3jQE48eAciYNiZtwNw7NBtWadHJ04dNohThw0C4AfHXc2sT+eU7d4aixee+zd/v/MvrLfBhuz0vc0B+PWZ5/L2lMncPOI6AHb/wRAOPDhZo77fehuw15B92XbAxjRrVsuFl17RdEew6zRA/JPUA7gN6AIsAm6IiCsknQ0MA2alWU+LiFHpMb8miSMLgZ9ExOg0fRBwBVAL/CkiLsp77Wz9JqUiaS1gTET0Tr//CmgOPA2cB7QHWgOjI+LYtJY4JiL+ImntNL3u2NtIAut9ua5Xs3LnaNl3/xLekS2Ld566vNxFsCy6tGvxckRs3lDna9mld6xx0JUF8029bPe815XUFegaEWMltQFeBoYA+wNzIuKSevnXB+4gWTO7G/AY0CfdPRHYBZgOvEhSWXs917XLUYOcl/F5IdAKuAUYEhHjJR0ObJ8l/6J6xy7C8zjNKpaAhmhBR8RMYGb6+UtJbwDd8xwyGLgzIuYBb0uaTBIsASZHxFQASXemeXMGyEoZpGkDzJTUHDio3IUxs4ZQuP8x7YPsKOmljG14zjMmrdBNgOfTpBMkvZKOTXRI07oD0zIOm56m5UrPqVJqYGeQ3PC7wKskAdPMqlyRNcjZxTTtJbUG7gFOiogvJF0HnAtE+vNS4Eiy93wG2SuEefsYV2iAjIh3gA0zvmf2HVyXJf/heY49vH5+M6sgSh6GaJBTJa3Le4C/RMS9ABHxYcb+EcBD6dfpQI+Mw9cAZqSfc6VnVSlNbDNrZEQSIAttBc+TtMNvBN6IiMsy0rtmZNsbqHsI5QFgqKSWknqRzJx5gWRQprekXpJaAEPTvDlVShPbzBqhBprmuA1wCPCqpLppfacBB0rqT9JMfgc4BiAiJqRTCF8HFgDHR8TCpDw6ARhNMs3npoiYkO/CDpBmVjINMRE8Ip4he7/iqDzHnA+cnyV9VL7j6nOANLOSUAP2QZaLA6SZlUjlP0pYiAOkmZVMlcdHB0gzKx3XIM3MsnAfpJlZHlVegXSANLPScRPbzCyHKo+PDpBmVhrugzQzy8nzIM3Mcqry+OgAaWYl4ia2mVl2yZILDpBmZlk5QJqZ5VDl8dEB0sxKxH2QZmbZydN8zMxyq/L46ABpZqVTU+URMmeAlNQ234ER8UXDF8fMGovG/qjhBJLVwjLvsO57AD1LWC4zawSqPD7mXhc7InpERM/0Z4963x0czawgSQW3Is7RQ9KTkt6QNEHST9P0VSWNkTQp/dkhTZekKyVNlvSKpE0zznVYmn+SpMMKXTtngKxXwKGSTks/ryFps2KOM7OmTSq8FWEB8LOIWA/YCjhe0vrAqcDjEdEbeDz9DrAb0DvdhgPXJWXRqsBZwJbAAOCsuqCaS8EAKelqYAeShbsB/gv8sajbMrMmS0CtVHArJCJmRsTY9POXwBtAd2AwcGua7VZgSPp5MHBbJJ4D2kvqCuwKjImITyLiU2AMMCjftYsZxd46IjaV9J+0gJ9IalHEcWbWlBXZhAY6Snop4/sNEXFD9lNqLWAT4Hlg9YiYCUkQldQ5zdYdmJZx2PQ0LVd6TsUEyPmSakgGZpC0GrCoiOPMrIkrsgk9OyI2L3wutQbuAU6KiC/yBN9sO+oPOGem51RMH+Q1aaE6Sfot8AxwcRHHmVkTJpJ5kIW2os4lNSeJQ3+JiHvT5A/TpjPpz4/S9OlAj4zD1wBm5EnPqWCAjIjbgNOBS4BPgP0i4s5Cx5mZ1dSo4FaIkqrijcAbEXFZxq4HgLqR6MOA+zPSD01Hs7cCPk+b4qOBgZI6pIMzA9O0nIp9kqYWmE9SHS1q5NvMmralGKUuZBuSQeJXJY1L004DLgLulnQU8B6wX7pvFLA7MJlkUPkIWDx+ci7wYprvnIj4JN+FCwZISb8BfgSMJKk1/1XSXyLiwuLvz8yaooZ41DAiniF7/yHATlnyB3B8jnPdBNxU7LWLqUEeDGwWEf8FkHQ+8DLgAGlmeVX5gzRFBch36+VrBkwtTXHMrLEQUFvlzxrme1nF5SR9jv8FJkganX4fSDKSbWaWW/HzICtWvhrka+nPCcDDGenPla44ZtaYVHl8zB0gI+LGFVkQM2tcGnUTu46kdYDzgfWBlerSI6JPCctlZo1AtTexi5nTeAtwM8k/CLsBdwOeKG5mBamIrZIVEyBXjojRABExJSJOJ3m7j5lZTlLDPWpYLsVM85mXPuozRdKxwPtA5wLHmJk16iUX6pwMtAZ+QtIX2Q44spSFMrPGocIriAUVDJAR8Xz68Uv+99JcM7O8ROU3oQvJN1F8JHnelRYR+5SkRA1ok/V68q/nry53MayeK56eUu4i2IrQcC+rKJt8NUhHFjNbLsUsqVDJ8k0Uf3xFFsTMGhdR/fMgi30fpJnZUqvyQWwHSDMrnSYTICW1jIh5pSyMmTUeUvU/i13MutgDJL0KTEq/byzpqpKXzMyqXt2yC/m2SlbMo4ZXAnsCHwNExHj8qKGZFdCQqxqWSzFN7JqIeLfeaNTCEpXHzBqRal/hr5gAOU3SACAk1QInAhNLWywzq3aSGn8fJHAccArQE/gQ2CpNMzPLqyH6ICXdJOkjSa9lpJ0t6X1J49Jt94x9v5Y0WdJbknbNSB+Upk2WdGox5S/mWeyPgKHFnMzMLFMDVSBvIXmy77Z66ZdHxCWZCZLWJ4lXGwDdgMck1b3c+xpgF2A68KKkByLi9XwXLuaN4iPI8kx2RAwvdKyZNV11gzTLKyL+KWmtIrMPBu5MpyS+LWkyMCDdNzkipgJIujPNu3wBEngs4/NKwN7AtCILa2ZNlaC2tKM0J0g6FHgJ+FlEfAp059sLC05P0+DbcWs6sGWhCxTTxL4r87uk24ExhY4zM1Nxiyp0lPRSxvcbIuKGAsdcB5xL0ro9F7iU5D212S4YZB9vyfm2sjrL8qhhL2DNZTjOzJqQpIldVNbZEbH50pw7Ij5cfJ2kG/Ch9Ot0oEdG1jWAGennXOk5FdMH+Sn/i7Q1wCdAUSNAZta0lWqaj6SuETEz/bo3UDfC/QDwV0mXkQzS9AZeIInXvSX1Ilk2Zijwo0LXyRsg07VoNk5PCLAoIgpWS83MlqIGmf880h3A9iRN8enAWcD2kvqTVN7eAY4BiIgJku4mGXxZABwfEQvT85wAjAZqgZsiYkKha+cNkBERkkZGxGbLeG9m1lQ10LPWEXFgluQb8+Q/n2T9rPrpo4BRS3PtYsaYXpC06dKc1MwMGvGz2JKaRcQC4HvAMElTgK9Ias4REQ6aZpaTKPk0n5LL18R+AdgUGLKCymJmjYqoKW6aT8XKFyAFEBFegs7MllqyJk25S7F88gXITpJOybUzIi4rQXnMrLFQ415yoRZoTfaZ6WZmeSV9kNUdPvIFyJkRcc4KK4mZNTqVPkpdSME+SDOzZVXl8TFvgNxphZXCzBod0YiXXIiIT1ZkQcyskVHjbmKbmS2zhnphbjk5QJpZyVR3eHSANLMSqvIKpAOkmZWGELVVHiEdIM2sZOQAaWaWXXWHRwdIMysVuQZpZpaVwH2QZma5VHd4dIA0sxKq8gqkA6SZlUbyLHZ1R0gHSDMrkcpflKuQan/ZhplVMKnwVvgcuknSR5Jey0hbVdIYSZPSnx3SdEm6UtJkSa9krsgq6bA0/yRJhxVTfgdIMyuJuiZ2oa0ItwCD6qWdCjweEb2Bx9PvALsBvdNtOHAdJAEVOAvYEhgAnFUXVPNxgDSz0hDU1BTeComIfwL1X784GLg1/Xwr/1t9dTBwWySeA9pL6grsCoyJiE8i4lNgDEsG3SW4D7ICff311+y8w7Z8M28eCxYuYO999uWMs37LscOOYuzLLxERrNunDyNuvIXWrVuXu7iNyt9+dypvPvcErduvxsk3/eNb+/55158Ydf1FnDHyBVZptyoAU8Y9x0PXnM/CBfNZpV0HjvnDHcx6byp/Pfeni4/7ZOZ77HL4SXxv3yNW6L1UAhVXQ+wo6aWM7zdExA0Fjlk9ImYCRMRMSZ3T9O7AtIx809O0XOl5OUBWoJYtW/LImCdo3bo18+fPZ8ftvsfAXXfjd5deTtu2bQH45c9P4bprr+YXvzy1wNlsaWy26z5sPeRg7r7oF99K/+yjGUx6+Rnad+62OG3unC+4/4qzOPKim2m/ejfmfPoxAJ16rs1PRzwIwKKFC7lg/23Y4HsDV9xNVIjkfZBFZZ0dEZs34GXrizzpebmJXYEkLa4Zzp8/nwXz5yNpcXCMCL6eO7fqH+OqRGtvPIBWbdsvkf7Qteez2zG/+taowrjHH2CD7+1K+9WToNm6w2pLHDd57L9ZrVtPOnQpWFlplFTEf8vow7TpTPrzozR9OtAjI98awIw86Xk5QFaohQsXsuVm/enZrTM77rwLA7bcEoDhRx3BWmt04a233uTHx59Y5lI2Da//6zHaduxCt3XW+1b67GnvMHfO51x/8o+46pjBvPzoyCWOHf/kw2y8454rqqgVp0YquC2jB4C6kejDgPsz0g9NR7O3Aj5Pm+KjgYGSOqSDMwPTtPzlX9bSWWnV1tby/MvjmPzOdF568QUmvJbMcLjhxpuZ+t4M+vVbj7/ffVeZS9n4ffP1XJ78y3UMPPykJfYtWriA9ye+xhEX/Ikjf3czT9x+NbOmvb14/4L53/DGvx9no+12X5FFrhh1TexCW8HzSHcAzwJ9JU2XdBRwEbCLpEnALul3gFHAVGAyMAL4MSxeY+tc4MV0O6eYdbfcB1nh2rdvz7bbbc+jjz7CBhtuCCTBc9/9D+DyS3/PoYc3vY7/FemTGe/xyQfT+MOwpBb4xawPuPKYwZxw7b2069SFldt1oEWrlWnRamV6fWcLZk55g049egHw1gtP0b33+rRZtWM5b6GMlqsJvVhEHJhj1xIrr0ZEAMfnOM9NwE1Lc+2S1SAlrSLpYUnjJb0m6QBJ70i6WNIL6bZumvcHkp6X9B9Jj0laPU0/W9Ktkh5Nj91H0u8kvSrpEUnNS1X+cpo1axafffYZAHPnzuWJxx+jT5++TJk8GUj6IB9+6EH69O1XzmI2CV3W7ssZ977AqXc8xal3PEXbTl34yfX302bVTqy/zc688+pLLFy4gG++nsu0N8bTec11Fx87/omH2HjHH5Sx9GVWxCTxSu9GL2UNchAwIyL2AJDUDrgY+CIiBkg6FPgDsCfwDLBVRISko4FfAj9Lz7MOsAOwPkk1+4cR8UtJI4E9gPtKeA9l8cHMmQw78jAWLlzIoljED/fdn91234Odtv8+X37xBUGw0UYbc+U115W7qI3OHeeexNTxz/PV559ywf7bsMvhP2WL3ffPmrfzmuvSZ4ttueLoPZBq2GL3/enSqw+QNM0nv/wv9jn5vBVZ/IrSGF53pqRGWoITS31IOkHvBh6KiKclvQPsGBFT09rfBxGxmqSNgEuBrkAL4O2IGCTpbGB+RJwvqQaYC6yUBtJzgE8i4g/1rjucZAY9PXr23GzilHdLcn+27K54ekq5i2BZnLrjui834HQb1ttok7h55JMF8323d4cGvW5DKlkTOyImApsBrwIXSjqzbldmtvTnVcDVEbERcAywUkaeeen5FpEEy7pjFpGlBhwRN0TE5hGxeaeOnRrsfsxsGaiIrYKVsg+yG/DfiPgzcAlQ99D4ARk/n00/twPeTz8X9RC5mVW+Es6DXCFK2Qe5EfB7SYuA+cBxwN+BlpKeJwnOdaNTZwN/k/Q+8BzQq4TlMrMVpMgnaSpWyQJkRIym3kTM9MmPayLit/Xy3s//Jnpmpp9d73vrXPvMrAI5QJqZLSnpYqzuCLlCA2RErLUir2dmZVQF8xwLcQ3SzErGAdLMLKvKH6UuxAHSzErGNUgzsyyqYB54QQ6QZlYy1f5SZwdIMyuZKo+PDpBmVjpVHh8dIM2sROQmtplZVsJNbDOznKo8PjpAmlkJVXmEdIA0s5JZjmVdK4KXfTWzkmmoF4qni/a9KmmcpJfStFUljZE0Kf3ZIU2XpCslTZb0iqRN8589NwdIMyudhl1yYYeI6J+xfs2pwOMR0Rt4PP0OsBvQO92GA8u8up0DpJmVRN37IEu45MJg4Nb0863AkIz02yLxHNBeUtdluYADpJmVhpIlFwptQEdJL2Vsw7OcLYBHJb2csX/1iJgJkP7snKZ3B6ZlHDs9TVtqHqQxs9IproI4u4hlX7eJiBmSOgNjJL25lFddpvWtXYM0sxIppoFdXASNiBnpz4+AkcAA4MO6pnP686M0+3SgR8bhawAzluUOHCDNrGSkwlvhc2gVSW3qPgMDgdeAB/jfMtGH8b+F/x4ADk1Hs7cCPq9rii8tN7HNrCQa8FHD1YGR6XPdzYC/RsQjkl4E7pZ0FPAesF+afxSwOzAZ+C9wxLJe2AHSzEqmIZZciIipwMZZ0j8GdsqSHsDxy31hHCDNrISq/EEaB0gzK50qj48OkGZWIn4fpJlZdn4fpJlZHlUeHx0gzax0XIM0M8vBfZBmZjlUd3h0gDSzEin2UcJK5gBpZiXjJraZWQ7VHR4dIM2shKq8AukAaWalstxLKpSdA6SZlYSfpDEzy8MB0swsBzexzcyy8TxIM7Ps3AdpZpaHm9hmZjm4BmlmlkOVx0cHSDMrnWp/FlvJComNk6RZwLvlLkcD6QjMLnchbAmN6feyZkR0aqiTSXqE5M+nkNkRMaihrtuQGnWAbEwkvRQRm5e7HPZt/r00bjXlLoCZWaVygDQzy8EBsnrcUO4CWFb+vTRi7oM0M8vBNUgzsxwcIM3McnCANDPLwQHSzN6bDUsAAAdeSURBVCwHB0gzsxwcIKuIqv3BVrMq42k+VUjSUKA3cCfwXkTMK3ORLCVpZ2B34H7gzYj4sMxFsuXgGmQVyKw5SjoA+BnQE/g9sLukNuUqm/2PpI2BC0j+Xh0FHCJpzfKWypaHA2SFk6RIq/mSOgOtgYMiYhjwKLAXsJOktmUsZpMnqQtJrf6SiDiJpHbfFdhPUq+yFs6WmZvYFaxecPwpcBzJP2rPRMSRafqxwC7AzcDD4V/oCidpEHAt8AHQLCIGZKTvBUwDLo+Ir8tXSlsWrkFWsIzguA2wJbAjcADQR9J5aZ4/AqOAsQ6OK56kfsARwN7A9sBKku4EiIhHgIeA+xwcq5NrkBUs7XtcF7gKWAQcEhEfS1qfpMbyn4g4uZxlbKrS30174EySf7iGR8Tz6b4XgI8iYs8yFtEagGuQFSZzQCYSk4ArgK+BgZJWi4jXgROB9SR18vSfFS/93XwKjACeAXaV9J103wCgp6RN/bupbq5BVpB6fY6HAl2AN0ma0LsCB5E02R6NiNmSWkTEN2UrcBMlaSCwA/A2cB/QATgG+AR4KCLGlbF41oBcg6wgGcHxJOBI4DOSKT3nAU8CtwEHAjtIqnFwXPEk7QlcCIwD9gSuB+YC1wDdgCGSWkvy361GwL/ECpD5l0lSH2BjYGegLcnKmasAZ5EEySuAf0fEojIUtUmT1AEYBOxHEhS7ABNJguNc4HLgroiY499P4+AmdgVJ58t9CHQmqY1cRDIAcDDwK2BkRJxWvhI2bek/ZJ1ImtS3A0OBAB4AZgI/8Gh14+J1sctI0tZAz4i4U9KJwE9Jaon/Jqk5PhMRCyTNBx4GrixfaZsuSduSTPr+JiJGSlqJ5DHCKZK2BB4HRjg4Nj4OkOXVAbgwnUu3BslAzI4kU3taASdJ6pSm7xIRH5StpE2UpC2AP5EMxmwt6YCIGCppLUl/Jvl9HRkRr5W1oFYSbmKXmaRdgMuA5yJimKSWJH1cnYBewD+BFyLivTIWs0mS9H2S38U/IuIfadqzwCMk3R+bAnM9at14eZCmzCJiDHA6MFjS0PTNPHcAX5BMG3nCwXHFk7Q28EPgMJIafZ1DgN4RMS8innVwbNzcxK4AEXG/pAUkzW3SPsmbgdYR8UW5y9fUSNoLOBvYA3gdOFnS08B4YE2SCfqrAp/68c7GzU3sCiJpN5J1lk+OiL+XuzxNkaT+wC3AgRHxRpr2Z5KpV88AtcCoiLivbIW0FcYBssKkfZJTImJqucvSFElaj2RK1bPA6iQvoHifpObYAzg2IkZLqo2IhWUrqK0QDpBmGSS1Bg4neWLpUpKJ4NsCk4B+wK+BgR61bhocIM2yqHvOXdLmJI94Hh8RT6bzVUdFxJQyF9FWAAdIsywk1QL9SV4rd0FE3F/mIlkZOECa5SBpFaBzRLxd99oyj1o3LQ6QZmY5eKK4mVkODpBmZjk4QJqZ5eAAaWaWgwNkEyBpoaRxkl6T9DdJKy/HubaX9FD6eS9Jp+bJ217Sj5fhGmdL+nmx6fXy3CJp36W41lqSPOnbsnKAbBrmRkT/iNgQ+AY4NnOnEkv9/0JEPBARF+XJ0h5Y6gBpVikcIJuep4F105rTG5KuBcYCPSQNlPSspLFpTbM1gKRBkt6U9AywT92JJB0u6er08+qSRkoan25bk7wzcZ209vr7NN8vJL0o6RVJv804128kvSXpMaBvoZuQNCw9z3hJ99SrFe8s6WlJE9NFtpBUK+n3Gdc+Znn/IK3xc4BsQiQ1A3YDXk2T+gK3RcQmwFck76XcOSI2BV4CTkmXFxgB/AD4PslCVdlcCTwVERuTvEh2AnAqyYs3+kfEL9LlUnsDA0ieUtlM0raSNiNZ32UTkgC8RRG3c29EbJFe7w3gqIx9awHbkbyu7I/pPRwFfB4RW6TnH5auAWSWk98H2TS0klT3YtengRtJFgV7NyKeS9O3AtYH/pU+NNKC5I02/YC3I2ISLH711/As19gROBQgfcvN5+kqgJkGptt/0u+tSQJmG5IFyf6bXuOBIu5pQ0nnkTTjWwOjM/bdna4qOEnS1PQeBgLfyeifbJdee2IR17ImygGyaZgbEf0zE9Ig+FVmEjAmIg6sl68/ycp9DUHAhRFxfb1rnLQM17gFGBIR4yUdTvJasjr1zxXptU+MiMxAiqS1lvK61oS4iW11ngO2kbQugKSV0zW63wR6SVonzXdgjuMfB45Lj62V1Bb4kqR2WGc0cGRG32Z3SZ1J1t3ZW1IrSW1ImvOFtAFmSmoOHFRv336SatIyrw28lV77uDQ/kvqkz1qb5eQapAEQEbPSmtgdShYOAzg9IiZKGg48LGk2yVu1N8xyip8CN0g6ClgIHBcRz0r6VzqN5h9pP+R6wLNpDXYOcHBEjJV0FzAOeJekG6CQM4Dn0/yv8u1A/BbwFMkLb4+NiK8l/Ymkb3Js+uKJWcCQ4v50rKnyyyrMzHJwE9vMLAcHSDOzHBwgzcxycIA0M8vBAdLMLAcHSDOzHBwgzcxy+H+ql9LjMiqGiQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Dictionary to store conditional probability and prior\n",
    "conditional_probability = dict()\n",
    "prior = dict()\n",
    "\n",
    "# Train using the training data\n",
    "trainMultinomialNaiveBayes(training_set, prior, conditional_probability)\n",
    "\n",
    "# Creating Confusion matrix\n",
    "# Generating True Positive, False Positive, False Negative, True Negative\n",
    "\n",
    "true_positives = 0\n",
    "true_negatives = 0\n",
    "false_positives = 0\n",
    "false_negatives = 0    \n",
    "\n",
    "for i in test_set:\n",
    "    test_set[i].setPredictedLabel(applyMultinomialNaiveBayes(test_set[i], prior, conditional_probability))\n",
    "    if test_set[i].getPredictedLabel() == test_set[i].getActualLabel() == \"spam\":\n",
    "        true_positives += 1\n",
    "    elif test_set[i].getPredictedLabel() == test_set[i].getActualLabel() == \"ham\":\n",
    "        true_negatives += 1\n",
    "    elif test_set[i].getPredictedLabel() != test_set[i].getActualLabel() and test_set[i].getPredictedLabel() == \"spam\":\n",
    "        false_positives += 1\n",
    "    else: #test_set[i].getPredictedLabel() != test_set[i].getActualLabel() and test_set[i].getPredictedLabel() == \"ham\":\n",
    "        false_negatives += 1\n",
    "\n",
    "# Computing performance metrics\n",
    "accuracy = float(true_positives+true_negatives) /float(true_positives+true_negatives+false_positives+false_negatives)    # (TP+TN)/(TP+TN+FP+FN)\n",
    "precision = float(true_positives) /float(true_positives+false_positives)        # (TP)/(TP+FP)\n",
    "recall = (true_positives) /float(true_positives+false_negatives)                # (TP)/(TP+FN)\n",
    "f1_score = 2* float((precision*recall) /float(precision+recall))                # 2*(P*R/P+R)\n",
    "\n",
    "# Printing performance metrics\n",
    "print (\"===============================CLASSIFICATION PERFORMANCE REPORT WITHOUT STOPWORDS REMOVED===============================\")\n",
    "print (\"No. of true positives:\", true_positives)\n",
    "print (\"No. of true negatives:\", true_negatives)\n",
    "print (\"No. of false positives:\", false_positives)\n",
    "print (\"No. of false negatives:\", false_negatives)\n",
    "print (\"Total documents in test set:\", len(test_set))\n",
    "print (\"Accuracy:\", 100.0*accuracy)\n",
    "print (\"Precision\", 100.0*precision)\n",
    "print (\"Recall:\", 100.0*recall)\n",
    "print (\"F1-score:\", 100.0*f1_score)\n",
    "cm = np.array([true_negatives, false_positives, false_negatives, true_positives]).reshape(2,2)\n",
    "plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Train and Test on dataset without stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============================CLASSIFICATION PERFORMANCE REPORT WITH STOPWORDS REMOVED===============================\n",
      "No. of true positives: 1467\n",
      "No. of true negatives: 3390\n",
      "No. of false positives: 282\n",
      "No. of false negatives: 33\n",
      "Total documents in test set: 5172\n",
      "Accuracy: 93.90951276102089\n",
      "Precision 83.87650085763293\n",
      "Recall: 97.8\n",
      "F1-score: 90.30470914127424\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAEYCAYAAAA+mm/EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3debyWc/7H8df7lBKnFJVCWcsyjOyGGYwlWcY2loydEQY/2wwx9t0wjG0Mxj6EGRpbQ4lByNZkyVLJlqIiCpWWz++P6zq5ne6tOve57/uc99Pjepz7/l7f67q+V0efvtt1fRURmJnZgmrKXQAzs0rlAGlmloMDpJlZDg6QZmY5OECameXgAGlmloMDpCGpjaRHJH0t6Z+LcZ4DJA1uyLKVi6RfSHqv3OWw8pLnQVYPSb8BTgbWAqYDI4GLImLYYp73IOB4YIuImLPYBa1wkgLoERFjy10Wq2yuQVYJSScDfwEuBpYHugN/BXZvgNOvDIxuDsGxGJJalrsMViEiwluFb8AywDfAPnnytCYJoBPS7S9A63TfNsB44BRgEjAROCzddx7wPTA7vcYRwLnAPzLOvQoQQMv0+6HAOJJa7AfAARnpwzKO2wJ4Bfg6/blFxr7/AhcAz6fnGQx0zHFvdeU/NaP8ewA7A6OBL4EzMvJvCrwIfJXmvQ5ole57Nr2Xb9P73S/j/KcBnwF31aWlx6yeXmPD9PsKwBRgm3L/v+GttJtrkNXhZ8CSwMA8ef4IbA70AtYnCRJnZuzvQhJoVyQJgtdL6hAR55DUSu+LiNqIuCVfQSQtDVwD7BQRbUmC4Mgs+ZYFHkvzLgdcCTwmabmMbL8BDgM6A62A3+e5dBeSP4MVgbOBm4EDgY2AXwBnS1otzTsXOAnoSPJntx3wO4CI2CrNs356v/dlnH9Zktp0v8wLR8T7JMHzbklLAbcBt0fEf/OU15oAB8jqsBwwJfI3gQ8Azo+ISRExmaRmeFDG/tnp/tkRMYik9rTmIpZnHrCupDYRMTEiRmXJswswJiLuiog5ETEAeBf4VUae2yJidETMAO4nCe65zCbpb50N3EsS/K6OiOnp9UcBPwWIiNciYnh63Q+BG4Gti7incyJiVlqeH4mIm4ExwEtAV5J/kKyJc4CsDl8AHQv0ja0AfJTx/aM0bf456gXY74DahS1IRHxL0iw9Gpgo6TFJaxVRnroyrZjx/bOFKM8XETE3/VwXwD7P2D+j7nhJPSU9KukzSdNIasgd85wbYHJEzCyQ52ZgXeDaiJhVIK81AQ6Q1eFFYCZJv1suE0iah3W6p2mL4ltgqYzvXTJ3RsQTEbEDSU3qXZLAUag8dWX6dBHLtDBuIClXj4hoB5wBqMAxeadzSKol6de9BTg37UKwJs4BsgpExNck/W7XS9pD0lKSlpC0k6Q/pdkGAGdK6iSpY5r/H4t4yZHAVpK6S1oGOL1uh6TlJe2W9kXOImmqz81yjkFAT0m/kdRS0n7AOsCji1imhdEWmAZ8k9Zuj6m3/3NgtQWOyu9q4LWI+C1J3+rfFruUVvEcIKtERFxJMgfyTGAy8AlwHPDvNMuFwKvAG8CbwIg0bVGuNQS4Lz3Xa/w4qNWQjIZPIBnZ3Zp0AKTeOb4Adk3zfkEyAr1rRExZlDItpN+TDABNJ6nd3ldv/7nAHZK+krRvoZNJ2h3oQ9KtAMnvYUNJBzRYia0ieaK4mVkOrkGameXgAGlmloMDpJlZDg6QZmY5NOmH8tWyTahV23IXw+pZb81u5S6CZfHGyBFTIqJTQ52vRbuVI+Ys8FDSAmLG5Cciok9DXbchNe0A2aotrdcsOIvDGtngZ64qdxEsiy7LtKr/5NNiiTkzivr7N3Pk9YWeciqbJh0gzayMJKhpUe5SLBYHSDMrHVX3MIcDpJmVjgo9Al/ZHCDNrETkGqSZWVbCfZBmZtnJTWwzs5zcxDYzy8E1SDOzLDwP0swsDzexzcyy8TQfM7PcatwHaWa2IM+DNDPLxU1sM7PcPM3HzCwH1yDNzLLwPEgzszzcxDYzy6b6B2mqu/RmVtmkwlvBU2hJSS9Lel3SKEnnpemrSnpJ0hhJ90lqlaa3Tr+PTfevknGu09P09yTtWOjaDpBmVhoS1LQsvBU2C9g2ItYHegF9JG0OXAZcFRE9gKnAEWn+I4CpEbEGcFWaD0nrAH2BnwB9gL9KyttJ6gBpZqXTADXISHyTfl0i3QLYFvhXmn4HsEf6eff0O+n+7SQpTb83ImZFxAfAWGDTfNd2gDSz0lFN4Q06Sno1Y+u3wGmkFpJGApOAIcD7wFcRMSfNMh5YMf28IvAJQLr/a2C5zPQsx2TlQRozK43ip/lMiYiN82WIiLlAL0ntgYHA2tmy1V05x75c6Tm5BmlmpdMATexMEfEV8F9gc6C9pLpK3krAhPTzeKBbcnm1BJYBvsxMz3JMVg6QZlYykgpuRZyjU1pzRFIbYHvgHeBpYO802yHAQ+nnh9PvpPufiohI0/umo9yrAj2Al/Nd201sMysJQVEBsAhdgTvSEeca4P6IeFTS28C9ki4E/gfckua/BbhL0liSmmNfgIgYJel+4G1gDnBs2nTPyQHSzEpDQg3wPsiIeAPYIEv6OLKMQkfETGCfHOe6CLio2Gs7QJpZyTRQDbJsHCDNrGQcIM3McnCANDPLQg3UB1lODpBmVjKuQZqZ5eAAaWaWgwOkmVk2wn2QZmbZiOIeJaxkDpBmVjIOkGZmuVR3fHSANLMSEdTUVPcLwxwgzaxk3MQ2M8vCgzS2yFq3asmTt5xIq1YtadmiBQOf/B8X/m0QN5zzGzZcpztCjP14EkeefRffzvie7l078LdzDqRjh1qmTvuOw/94B59O+gqAA361Gf1/m6xgeenfn+DuR14q5601GZ+O/4Tjjz6cyZ9/hmpqOOjQ33LkMcfz1hsjOfWk45g1ayYtWrTk0iuvZcONNuGB++/hur9cAcDSS9dy2ZXX8pP11i/zXZRZdcdHB8hymfX9HPr0u4ZvZ3xPy5Y1PHXryQx+/m1OveJBpn87E4DLTtmLY/puzRW3DeGSk/bk7sde5u5HXmLrTXpy/vG7ccRZd9Kh3VL8sd9ObHnAn4gIXrjnNB777xt8NX1Gme+w+rVs2ZJzL/wTP+21Ad9Mn07vrTdjq19uxwVnn8Ep/c9kux368OTg/3DB2acz8LEn6b7yqgx8bCjtO3Rg6JDH+f0Jv+M/Tz1f7tsonybQB1ndpa9y3874HoAlWragZcsWRMT84AiwZOslSN4UD2ut1pX/vvQeAM+8Mppdt1kPgB22WJuhw99l6rTv+Gr6DIYOf5feW67TyHfSNC3fpSs/7ZW8p7W2bVt6rLkWn02YgCSmT5sGwPRpX9OlS1cANtnsZ7Tv0AGAjTbejIkTPi1PwStIQyy5UE6uQZZRTY144Z7TWL1bJ26871leeesjAG4890B2/Pk6vDvuM/pf+SAAb47+lD2268X1A/7L7tuuT7vaNiy7zNKs0Kk94z+fOv+cn076ihU6tS/L/TRlH3/0IW+98Tobbrwp5196BfvvtSvnn9WfefPm8cjgZxbIf89dt7Ht9juWoaQVprLjX0GNWoOUtIqktxrzmpVs3rxg876XssaOZ7LxuiuzzupJTeSoc//Bar3/yLsffMbevTcC4PSrBvKLjdbgxQGn8YuN1uDTz6cyZ+7crIvCRf6VLG0hffvNN/z2oP04/5IraNuuHXfcchPnXXw5I94ex3kXX87Jxx31o/zDnv0vA+66jTPPv7hMJa4MkqipqSm4VbLKLl0z8fU3M3j21TH03uKHpvG8ecG/Bo9gj+16ATBx8tf0/f3f+dn+l3HOdY8AMO2bmXw66StWWr7D/ONW7NyeiZO/btwbaMJmz57NEQftx1777s8uu+0JwP0D7pr/ebc99+Z/I16Zn//tt97glOOP5vYBD7DsssuVpcyVpNqb2OUIkC0k3SxplKTBktpIOlLSK5Jel/SApKUAJN0u6QZJT0saJ2lrSbdKekfS7WUoe4Pp2KGWZWrbAElf47abrcnojz5ntW4d5+fZZav1GP3h5wAs137p+f8z/eHwHbnjoeEADHnhHbb/2Vq0b9uG9m3bsP3P1mLIC+808t00TRHBScf1o8eaa3H0cSfOT+/SpSsvDHsWgGHPPM1qq60BwPhPPubwA/fjuptuY/U1epalzJWm2gNkOfogewD7R8SR6RKMvwYejIibAdIlHI8Ark3zdwC2BXYDHgG2BH4LvCKpV0SMzDy5pH5APwCWqC393SyiLh3bcfP5B9GipoaaGvHAkBH857lRDL31RNou3QYp6Xf8v4vvA2CrjXtw/vG7EQHDRozlxEvuB2DqtO+45ObHGfaPUwG4+KbHmTrtu7LdV1Py8vAX+Ne9d7P2T9Zlu59vDMDpZ1/AFdf8jbNOO5k5c+fQuvWSXH71DQBcedlFTP3yC/qfcjwALVq0ZPAzw8tW/orQAPFPUjfgTqALMA+4KSKulnQucCQwOc16RkQMSo85nSSOzAX+LyKeSNP7AFcDLYC/R8Slea9dN0raGCStAgyJiB7p99OAJYDngAuB9kAt8EREHJ3WEodExN2SVkvT6469kySw/jvX9WqW6hyt19y3hHdki+LDZ64qdxEsiy7LtHotIjZuqPO17tIjVjrgmoL5xl25c97rSuoKdI2IEZLaAq8BewD7At9ExBX18q8DDCBZEnYF4Emgrko/GtgBGA+8QlJZezvXtctRg5yV8Xku0Aa4HdgjIl6XdCiwTZb88+odOw+PwptVLEHWQcSFFRETgYnp5+mS3gFWzHPI7sC9ETEL+EDSWH5YP3tsup42ku5N8+YMkJUySNMWmChpCeCAchfGzBpC4f7HtA+yo6RXM7Z+Oc+YtEI3AOoeFztO0hvp2ETdaOWKwCcZh41P03Kl51QpNbCzSG74I+BNkoBpZlWuyBrklGKa9pJqgQeAEyNimqQbgAuASH/+GTic7D2fQfYKYd4+xkYNkBHxIbBuxvfMvoMbsuQ/NM+xh9bPb2YVRMnDEA1yqqR1+QBwd0Q8CBARn2fsvxl4NP06HuiWcfhKwIT0c670rCqliW1mTYxIAmShreB5knb4LcA7EXFlRnrXjGx7AnUPoTwM9JXUWtKqJDNnXiYZlOkhaVVJrYC+ad6cKqWJbWZNUANNc9wSOAh4U1LdtL4zgP0l9SJpJn8IHAUQEaPSKYRvA3OAYyNiblIeHQc8QTLN59aIGJXvwg6QZlYyDTERPCKGkb1fcVCeYy4CLsqSPijfcfU5QJpZSagB+yDLxQHSzEqk8h8lLMQB0sxKpsrjowOkmZWOa5BmZlm4D9LMLI8qr0A6QJpZ6biJbWaWQ5XHRwdIMysN90GameXkeZBmZjlVeXx0gDSzEnET28wsu2TJBQdIM7OsHCDNzHKo8vjoAGlmJeI+SDOz7ORpPmZmuVV5fHSANLPSqanyCJkzQEpql+/AiJjW8MUxs6aiqT9qOIpktbDMO6z7HkD3EpbLzJqAKo+PudfFjohuEdE9/dmt3ncHRzMrSFLBrYhzdJP0tKR3JI2SdEKavqykIZLGpD87pOmSdI2ksZLekLRhxrkOSfOPkXRIoWvnDJD1CthX0hnp55UkbVTMcWbWvEmFtyLMAU6JiLWBzYFjJa0D9AeGRkQPYGj6HWAnoEe69QNuSMqiZYFzgM2ATYFz6oJqLgUDpKTrgF+SLNwN8B3wt6Juy8yaLQEtpIJbIRExMSJGpJ+nA+8AKwK7A3ek2e4A9kg/7w7cGYnhQHtJXYEdgSER8WVETAWGAH3yXbuYUewtImJDSf9LC/ilpFZFHGdmzVmRTWigo6RXM77fFBE3ZT+lVgE2AF4Clo+IiZAEUUmd02wrAp9kHDY+TcuVnlMxAXK2pBqSgRkkLQfMK+I4M2vmimxCT4mIjQufS7XAA8CJETEtT/DNtqP+gHNmek7F9EFenxaqk6TzgGHAZUUcZ2bNmEjmQRbaijqXtARJHLo7Ih5Mkz9Pm86kPyel6eOBbhmHrwRMyJOeU8EAGRF3AmcCVwBfAvtExL2FjjMzq6lRwa0QJVXFW4B3IuLKjF0PA3Uj0YcAD2WkH5yOZm8OfJ02xZ8AekvqkA7O9E7Tcir2SZoWwGyS6mhRI99m1rwtxCh1IVuSDBK/KWlkmnYGcClwv6QjgI+BfdJ9g4CdgbEkg8qHwfzxkwuAV9J850fEl/kuXDBASvoj8BtgIEmt+R5Jd0fEJcXfn5k1Rw3xqGFEDCN7/yHAdlnyB3BsjnPdCtxa7LWLqUEeCGwUEd8BSLoIeA1wgDSzvKr8QZqiAuRH9fK1BMaVpjhm1lQIaFHlzxrme1nFVSR9jt8BoyQ9kX7vTTKSbWaWW/HzICtWvhrkW+nPUcBjGenDS1ccM2tKqjw+5g6QEXFLYxbEzJqWJt3EriNpdeAiYB1gybr0iOhZwnKZWRNQ7U3sYuY03g7cRvIPwk7A/YAniptZQSpiq2TFBMilIuIJgIh4PyLOJHm7j5lZTlLDPWpYLsVM85mVPurzvqSjgU+BzgWOMTNr0ksu1DkJqAX+j6Qvchng8FIWysyahgqvIBZUMEBGxEvpx+n88NJcM7O8ROU3oQvJN1F8IHnelRYRe5WkRA1og7W78/xL15W7GFbP1c+9X+4iWGNouJdVlE2+GqQji5ktlmKWVKhk+SaKD23MgphZ0yKqfx5kse+DNDNbaFU+iO0AaWal02wCpKTWETGrlIUxs6ZDqv5nsYtZF3tTSW8CY9Lv60u6tuQlM7OqV7fsQr6tkhXzqOE1wK7AFwAR8Tp+1NDMCmjIVQ3LpZgmdk1EfFRvNGpuicpjZk1Ita/wV0yA/ETSpkBIagEcD4wubbHMrNpJqvo+yGIC5DEkzezuwOfAk2mamVleFd6CLqhgDTgiJkVE34jomG59I2JKYxTOzKpbjQpvhUi6VdIkSW9lpJ0r6VNJI9Nt54x9p0saK+k9STtmpPdJ08ZK6l9M+Yt5o/jNZHkmOyL6FXMBM2ue6gZpGsDtJI8+31kv/aqIuOJH15TWAfoCPwFWAJ6UVLf6wfXADsB44BVJD0fE2/kuXEwT+8mMz0sCewKfFHGcmTVnghYNMEoTEc9KWqXI7LsD96Zztj+QNBbYNN03NiLGAUi6N827eAEyIu7L/C7pLmBIkYU1s2ZMxS2q0FHSqxnfb4qIm4o47jhJBwOvAqdExFRgRX688ur4NA1+XLEbD2xW6AKLEt9XBVZehOPMrBlJmthF9UFOiYiNM7ZiguMNwOpAL2Ai8OeMy9YXedLzKqYPcmrGiWqAL4GiOjjNrHkr1TSfiPi87nM6TvJo+nU80C0j60rAhPRzrvSc8gbIdC2a9UnWoQGYFxEFo66ZWV0NsiTnlrpGxMT0655A3Qj3w8A9kq4kGaTpAbycFqeHpFVJ4llf4DeFrpM3QEZESBoYERst2m2YWbPVQM9aSxoAbEPSVzkeOAfYRlIvktbth8BRABExStL9JIMvc4BjI2Juep7jgCeAFsCtETGq0LWLGcV+WdKGETFiYW/MzJq3hpjmExH7Z0m+JU/+i0gWGKyfPggYtDDXzrcmTcuImAP8HDhS0vvAtyRV1YiIDRfmQmbWvIiGmeZTTvlqkC8DGwJ7NFJZzKxJETXFTfOpWPkCpAAiwkvQmdlCS9akKXcpFk++ANlJ0sm5dkbElSUoj5k1FUU+a13J8gXIFkAt2SdYmpnllfRBVnf4yBcgJ0bE+Y1WEjNrcir9jeGFFOyDNDNbVFUeH/MGyO0arRRm1uSIJrzkQkR82ZgFMbMmRk27iW1mtsga8IW5ZeMAaWYlU93h0QHSzEqoyiuQDpBmVhpCtKjyCOkAaWYlIwdIM7Psqjs8OkCaWanINUgzs6wE7oM0M8ulusOjA6SZlVCVVyAdIM2sNJJnsas7QjpAmlmJqOofNaz2l22YWQWTCm+Fz6FbJU2S9FZG2rKShkgak/7skKZL0jWSxkp6Q9KGGccckuYfI+mQYsrvAGlmJVHXxC60FeF2oE+9tP7A0IjoAQxNvwPsBPRIt37ADZAEVJL1tDcDNgXOqQuq+ThAmllpCGpqCm+FRMSzQP3XL+4O3JF+voMfVl/dHbgzEsOB9pK6AjsCQyLiy4iYCgxhwaC7APdBVqCZM2ey/S+34vtZs5gzdw577rU3Z51zHkcfeQQjXnuViGCNnj25+Zbbqa2tLXdxm5R//qk/7w5/itr2y3HSrf/50b5n7/s7g268lLMGvszSyywLwPsjh/Po9Rcxd85sll6mA0f9ZQCTPx7HPRecMP+4Lyd+zA6HnsjP9z6sUe+lEqi4GmJHSa9mfL8pIm4qcMzyETERICImSuqcpq8IfJKRb3yalis9LwfICtS6dWseH/IUtbW1zJ49m223/jm9d9yJP/35Ktq1awfAqb8/mRv+eh1/OLV/gbPZwthox73YYo8Duf/SP/wo/atJExjz2jDad15hftqMb6bx0NXncPilt9F++RX4ZuoXAHTqvhon3PwIAPPmzuXifbfkJz/v3Xg3USGS90EWlXVKRGzcgJetL/Kk5+UmdgWSNL9mOHv2bObMno2k+cExIpg5Y0bVP8ZViVZbf1PatGu/QPqjf72InY467UejCiOHPsxPfr4j7ZdPgmZth+UWOG7siBdYboXudOhSsLLSJKmI/xbR52nTmfTnpDR9PNAtI99KwIQ86Xk5QFaouXPnstlGvei+Qme23X4HNt1sMwD6HXEYq6zUhffee5ffHXt8mUvZPLz9/JO069iFFVZf+0fpUz75kBnffM2NJ/2Ga4/andcGD1zg2Neffoz1t921sYpacWqkgtsiehioG4k+BHgoI/3gdDR7c+DrtCn+BNBbUod0cKZ3mpa//ItaOiutFi1a8NJrIxn74XhefeVlRr2VzHC46ZbbGPfxBNZaa23+df99ZS5l0/f9zBk8ffcN9D70xAX2zZs7h09Hv8VhF/+dw/90G0/ddR2TP/lg/v45s7/nnReGst7WOzdmkStGXRO70FbwPNIA4EVgTUnjJR0BXArsIGkMsEP6HWAQMA4YC9wM/A7mr7F1AfBKup1fzLpb7oOscO3bt2errbdh8ODH+cm66wJJ8Nx73/246s+Xc/Chza/jvzF9OeFjvvzsE/5yZFILnDb5M645aneO++uDLNOpC0st04FWbZaiVZulWPWnmzDx/Xfo1G1VAN57+RlW7LEObZftWM5bKKPFakLPFxH759i1wMqrERHAsTnOcytw68Jcu2Q1SElLS3pM0uuS3pK0n6QPJV0m6eV0WyPN+ytJL0n6n6QnJS2fpp8r6Q5Jg9Nj95L0J0lvSnpc0hKlKn85TZ48ma+++gqAGTNm8NTQJ+nZc03eHzsWSPogH3v0EXquuVY5i9ksdFltTc568GX6D3iG/gOeoV2nLvzfjQ/RdtlOrLPl9nz45qvMnTuH72fO4JN3XqfzymvMP/b1px5l/W1/VcbSl1kRk8QrvRu9lDXIPsCEiNgFQNIywGXAtIjYVNLBwF+AXYFhwOYREZJ+C5wKnJKeZ3Xgl8A6JNXsX0fEqZIGArsA/y7hPZTFZxMncuThhzB37lzmxTx+vfe+7LTzLmy3zS+YPm0aQbDeeutzzfU3lLuoTc6AC05k3Osv8e3XU7l43y3Z4dAT2GTnfbPm7bzyGvTcZCuu/u0uSDVssvO+dFm1J5A0zce+9jx7nXRhYxa/ojSF150pqZGW4MRST5JO0PuBRyPiOUkfAttGxLi09vdZRCwnaT3gz0BXoBXwQUT0kXQuMDsiLpJUA8wAlkwD6fnAlxHxl3rX7Ucyg55u3btvNPr9j0pyf7born7u/XIXwbLov+0arzXgdBvWXm+DuG3g0wXz/axHhwa9bkMqWRM7IkYDGwFvApdIOrtuV2a29Oe1wHURsR5wFLBkRp5Z6fnmkQTLumPmkaUGHBE3RcTGEbFxp46dGux+zGwRqIitgpWyD3IF4LuI+AdwBVD30Ph+GT9fTD8vA3yafi7qIXIzq3wlnAfZKErZB7kecLmkecBs4BjgX0BrSS+RBOe60alzgX9K+hQYDqxawnKZWSMp8kmailWyABkRT1BvImb65Mf1EXFevbwP8cNEz8z0c+t9r821z8wqkAOkmdmCki7G6o6QjRogI2KVxryemZVRFcxzLMQ1SDMrGQdIM7OsKn+UuhAHSDMrGdcgzcyyqIJ54AU5QJpZyVT7S50dIM2sZKo8PjpAmlnpVHl8dIA0sxKRm9hmZlkJN7HNzHKq8vjoAGlmJVTlEdIB0sxKZjGWda0IXvbVzEqmoV4oni7a96akkZJeTdOWlTRE0pj0Z4c0XZKukTRW0huSNsx/9twcIM2sdBp2yYVfRkSvjPVr+gNDI6IHMDT9DrAT0CPd+gGLvLqdA6SZlUTd+yBLuOTC7sAd6ec7gD0y0u+MxHCgvaSui3IBB0gzKw0lSy4U2oCOkl7N2PplOVsAgyW9lrF/+YiYCJD+7Jymrwh8knHs+DRtoXmQxsxKp7gK4pQiln3dMiImSOoMDJH07kJedZHWt3YN0sxKpJgGdnERNCImpD8nAQOBTYHP65rO6c9JafbxQLeMw1cCJizKHThAmlnJSIW3wufQ0pLa1n0GegNvAQ/zwzLRh/DDwn8PAweno9mbA1/XNcUXlpvYZlYSDfio4fLAwPS57pbAPRHxuKRXgPslHQF8DOyT5h8E7AyMBb4DDlvUCztAmlnJNMSSCxExDlg/S/oXwHZZ0gM4drEvjAOkmZVQlT9I4wBpZqVT5fHRAdLMSsTvgzQzy87vgzQzy6PK46MDpJmVjmuQZmY5uA/SzCyH6g6PDpBmViLFPkpYyRwgzaxk3MQ2M8uhusOjA6SZlVCVVyAdIM2sVBZ7SYWyc4A0s5LwkzRmZnk4QJqZ5eAmtplZNp4HaWaWnfsgzczycBPbzCwH1yDNzHKo8vjoAGlmpVPtz2IrWSGxaZI0Gfio3OVoIB2BKeUuhC2gKf1eVo6ITg11MkmPk/z5FDIlIvo01HUbUpMOkE2JpFcjYuNyl8N+zL+Xpq2m3AUwM6tUDpBmZjk4QFaPm8pdAMvKv5cmzJ4HQWwAAAd2SURBVH2QZmY5uAZpZpaDA6SZWQ4OkGZmOThAmpnl4ABpZpaDA2QVUbU/2GpWZTzNpwpJ6gv0AO4FPo6IWWUukqUkbQ/sDDwEvBsRn5e5SLYYXIOsApk1R0n7AacA3YHLgZ0ltS1X2ewHktYHLib5e3UEcJCklctbKlscDpAVTpIireZL6gzUAgdExJHAYGA3YDtJ7cpYzGZPUheSWv0VEXEiSe2+K7CPpFXLWjhbZG5iV7B6wfEE4BiSf9SGRcThafrRwA7AbcBj4V9oo5PUB/gr8BnQMiI2zUjfDfgEuCoiZpavlLYoXIOsYBnBcUtgM2BbYD+gp6QL0zx/AwYBIxwcG5+ktYDDgD2BbYAlJd0LEBGPA48C/3ZwrE6uQVawtO9xDeBaYB5wUER8IWkdkhrL/yLipHKWsblKfzftgbNJ/uHqFxEvpfteBiZFxK5lLKI1ANcgK0zmgEwkxgBXAzOB3pKWi4i3geOBtSV18vSfxpf+bqYCNwPDgB0l/TTdtynQXdKG/t1UN9cgK0i9PseDgS7AuyRN6B2BA0iabIMjYoqkVhHxfdkK3ExJ6g38EvgA+DfQATgK+BJ4NCJGlrF41oBcg6wgGcHxROBw4CuSKT0XAk8DdwL7A7+UVOPg2Pgk7QpcAowEdgVuBGYA1wMrAHtIqpXkv1tNgH+JFSDzL5OknsD6wPZAO5KVM5cGziEJklcDL0TEvDIUtVmT1AHoA+xDEhS7AKNJguMM4Crgvoj4xr+fpsFN7AqSzpf7HOhMUhu5lGQA4EDgNGBgRJxRvhI2b+k/ZJ1ImtR3AX2BAB4GJgK/8mh10+J1sctI0hZA94i4V9LxwAkktcQXSGqOwyJijqTZwGPANeUrbfMlaSuSSd/fR8RASUuSPEb4vqTNgKHAzQ6OTY8DZHl1AC5J59KtRDIQsy3J1J42wImSOqXpO0TEZ2UraTMlaRPg7ySDMVtI2i8i+kpaRdI/SH5fh0fEW2UtqJWEm9hlJmkH4EpgeEQcKak1SR9XJ2BV4Fng5Yj4uIzFbJYk/YLkd/GfiPhPmvYi8DhJ98eGwAyPWjddHqQps4gYApwJ7C6pb/pmngHANJJpI085ODY+SasBvwYOIanR1zkI6BERsyLiRQfHps1N7AoQEQ9JmkPS3Cbtk7wNqI2IaeUuX3MjaTfgXGAX4G3gJEnPAa8DK5NM0F8WmOrHO5s2N7EriKSdSNZZPiki/lXu8jRHknoBtwP7R8Q7ado/SKZeDQNaAIMi4t9lK6Q1GgfICpP2Sb4fEePKXZbmSNLaJFOqXgSWJ3kBxackNcduwNER8YSkFhExt2wFtUbhAGmWQVItcCjJE0t/JpkIvhUwBlgLOB3o7VHr5sEB0iyLuufcJW1M8ojnsRHxdDpfdVBEvF/mIlojcIA0y0JSC6AXyWvlLo6Ih8pcJCsDB0izHCQtDXSOiA/qXlvmUevmxQHSzCwHTxQ3M8vBAdLMLAcHSDOzHBwgzcxycIBsBiTNlTRS0luS/ilpqcU41zaSHk0/7yapf5687SX9bhGuca6k3xebXi/P7ZL2XohrrSLJk74tKwfI5mFGRPSKiHWB74GjM3cqsdD/L0TEwxFxaZ4s7YGFDpBmlcIBsvl5DlgjrTm9I+mvwAigm6Tekl6UNCKtadYCSOoj6V1Jw4C96k4k6VBJ16Wfl5c0UNLr6bYFyTsTV09rr5en+f4g6RVJb0g6L+Ncf5T0nqQngTUL3YSkI9PzvC7pgXq14u0lPSdpdLrIFpJaSLo849pHLe4fpDV9DpDNiKSWwE7Am2nSmsCdEbEB8C3Jeym3j4gNgVeBk9PlBW4GfgX8gmShqmyuAZ6JiPVJXiQ7CuhP8uKNXhHxh3S51B7ApiRPqWwkaStJG5Gs77IBSQDepIjbeTAiNkmv9w5wRMa+VYCtSV5X9rf0Ho4Avo6ITdLzH5muAWSWk98H2Ty0kVT3YtfngFtIFgX7KCKGp+mbA+sAz6cPjbQieaPNWsAHETEG5r/6q1+Wa2wLHAyQvuXm63QVwEy90+1/6fdakoDZlmRBsu/SazxcxD2tK+lCkmZ8LfBExr7701UFx0gal95Db+CnGf2Ty6TXHl3EtayZcoBsHmZERK/MhDQIfpuZBAyJiP3r5etFsnJfQxBwSUTcWO8aJy7CNW4H9oiI1yUdSvJasjr1zxXptY+PiMxAiqRVFvK61oy4iW11hgNbSloDQNJS6Rrd7wKrSlo9zbd/juOHAsekx7aQ1A6YTlI7rPMEcHhG3+aKkjqTrLuzp6Q2ktqSNOcLaQtMlLQEcEC9fftIqknLvBrwXnrtY9L8SOqZPmttlpNrkAZARExOa2IDlCwcBnBmRIyW1A94TNIUkrdqr5vlFCcAN0k6ApgLHBMRL0p6Pp1G85+0H3Jt4MW0BvsNcGBEjJB0HzAS+IikG6CQs4CX0vxv8uNA/B7wDMkLb4+OiJmS/k7SNzkiffHEZGCP4v50rLnyyyrMzHJwE9vMLAcHSDOzHBwgzcxycIA0M8vBAdLMLAcHSDOzHBwgzcxy+H968O1Q+9BC+AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set the stop words\n",
    "def setStopWords():\n",
    "    stops = []\n",
    "    with open('stopwords.txt', 'r') as txt:\n",
    "        stops = (txt.read().splitlines())\n",
    "        stops = set(stops)\n",
    "    return stops\n",
    "\n",
    "# Remove stop words from data set and store in dictionary\n",
    "def removeStopWords(stops, data_set):\n",
    "    filtered_data_set = copy.deepcopy(data_set)\n",
    "    for i in stops:\n",
    "        for j in filtered_data_set:\n",
    "            if i in filtered_data_set[j].getWordFreqs():\n",
    "                del filtered_data_set[j].getWordFreqs()[i]\n",
    "    return filtered_data_set\n",
    "\n",
    "# Set the stop words list\n",
    "stop_words = setStopWords()\n",
    "\n",
    "# Filtered sets without stop words\n",
    "filtered_training_set = dict()\n",
    "filtered_test_set = dict()\n",
    "\n",
    "# Set up data sets without stop words\n",
    "filtered_training_set = removeStopWords(stop_words, training_set)\n",
    "filtered_test_set = removeStopWords(stop_words, test_set)\n",
    "\n",
    "# Dictionary to store conditional probability and prior\n",
    "filtered_conditional_probability = dict()\n",
    "filtered_prior = dict()\n",
    "\n",
    "# Train using the training data\n",
    "trainMultinomialNaiveBayes(filtered_training_set, filtered_prior, filtered_conditional_probability)\n",
    "\n",
    "# Creating Confusion matrix\n",
    "# Generating True Positive, False Positive, False Negative, True Negative\n",
    "true_positives = 0\n",
    "true_negatives = 0\n",
    "false_positives = 0\n",
    "false_negatives = 0    \n",
    "\n",
    "for i in filtered_test_set:\n",
    "    filtered_test_set[i].setPredictedLabel(applyMultinomialNaiveBayes(filtered_test_set[i], filtered_prior, filtered_conditional_probability))\n",
    "    if filtered_test_set[i].getPredictedLabel() == filtered_test_set[i].getActualLabel() == \"spam\":\n",
    "        true_positives += 1\n",
    "    elif filtered_test_set[i].getPredictedLabel() == filtered_test_set[i].getActualLabel() == \"ham\":\n",
    "        true_negatives += 1\n",
    "    elif filtered_test_set[i].getPredictedLabel() != filtered_test_set[i].getActualLabel() and filtered_test_set[i].getPredictedLabel() == \"spam\":\n",
    "        false_positives += 1\n",
    "    else: #filtered_test_set[i].getPredictedLabel() != filtered_test_set[i].getActualLabel() and filtered_test_set[i].getPredictedLabel() == \"ham\":\n",
    "        false_negatives += 1\n",
    "\n",
    "# Computing performance metrics\n",
    "accuracy = float(true_positives+true_negatives) /float(true_positives+true_negatives+false_positives+false_negatives)    # (TP+TN)/(TP+TN+FP+FN)\n",
    "precision = float(true_positives) /float(true_positives+false_positives)        # (TP)/(TP+FP)\n",
    "recall = (true_positives) /float(true_positives+false_negatives)                # (TP)/(TP+FN)\n",
    "f1_score = 2* float((precision*recall) /float(precision+recall))                # 2*(P*R/P+R)\n",
    "\n",
    "# Printing performance metrics\n",
    "print (\"===============================CLASSIFICATION PERFORMANCE REPORT WITH STOPWORDS REMOVED===============================\")\n",
    "print (\"No. of true positives:\", true_positives)\n",
    "print (\"No. of true negatives:\", true_negatives)\n",
    "print (\"No. of false positives:\", false_positives)\n",
    "print (\"No. of false negatives:\", false_negatives)\n",
    "print (\"Total documents in test set:\", len(filtered_test_set))\n",
    "print (\"Accuracy:\", 100.0*accuracy)\n",
    "print (\"Precision\", 100.0*precision)\n",
    "print (\"Recall:\", 100.0*recall)\n",
    "print (\"F1-score:\", 100.0*f1_score)\n",
    "cm = np.array([true_negatives, false_positives, false_negatives, true_positives]).reshape(2,2)\n",
    "plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}