LIAR_dataset_classifying_using_svm.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "LIAR dataset classifying using svm.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Interpause/pseudo-text/blob/master/LIAR_dataset_classifying_using_svm.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5x0RHPSwpIjV",
        "colab_type": "text"
      },
      "source": [
        "# Imports & setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ozUqLnAISSFI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import nltk\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "from sklearn.svm import SVC\n",
        "from sklearn import metrics,model_selection\n",
        "\n",
        "from collections import OrderedDict\n",
        "import string, json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lz1SkOvjTeIR",
        "colab_type": "code",
        "outputId": "db05120f-eb25-4b42-8208-bfad5d00ede0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        }
      },
      "source": [
        "#Online instance only.\n",
        "from google.colab import drive #drive.flush_and_unmount() #drive.flush()\n",
        "import os\n",
        "drive.mount('/content/drive')\n",
        "os.chdir('/content/drive/My Drive/CSIT Internship/code')\n",
        "nltk.download('all',quiet=True)\n",
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n",
            "Thu Nov 28 01:41:35 2019       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.33.01    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   38C    P0    26W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ACQz9f2QpRVQ",
        "colab_type": "text"
      },
      "source": [
        "# Cleaning Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-9sZYE3SSFW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# List of all words to remove from statements\n",
        "removables = []\n",
        "removables += nltk.corpus.stopwords.words('english')\n",
        "#removables += [w.lower() for w in nltk.corpus.names.words()]\n",
        "removables += ['hi','day','go']\n",
        "replacements = {}\n",
        "replacements = OrderedDict({'barack obama':'obama','donald trump':'trump','health care':'healthcare','hillary clinton':'hillary','president obama':'obama','barack':'obama','donald':'trump','gov ':'government ','said':'say'})\n",
        "lemmatizer = nltk.stem.WordNetLemmatizer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZZvVBqAwSSFb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#much info was lost\n",
        "#some of these could be specified as options to the Tfidf vectorizer instead\n",
        "def filterSent(s):\n",
        "    s = s.lower()\n",
        "    for rep,val in replacements.items(): s = s.replace(rep,val)\n",
        "    s = nltk.word_tokenize(s) #split into list of words intelligently\n",
        "    s = tuple(w for w in s if w not in removables) #remove removables\n",
        "    s = tuple(lemmatizer.lemmatize(w) for w in s if w.isalpha()) #supposed to lemmatize words (aka get basic form)\n",
        "    return ' '.join(s)\n",
        "filterSent(\"hi there my name is jOhn henry battery car dead better programming\")\n",
        "\n",
        "#applied to dataset to reformat data\n",
        "formatter = lambda x:(filterSent(x.statement),liar_score[x.label])\n",
        "\n",
        "def printScore(y_pred,y_test):\n",
        "    #assuming balanced, otherwise using balanced_accuracy_score()\n",
        "    print(\"Accuracy: {0:.1%}\".format(metrics.accuracy_score(y_test, y_pred)))\n",
        "    #TODO add more"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XXpLLN9JpdnY",
        "colab_type": "text"
      },
      "source": [
        "# LIAR dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iLf1yZEXSSFn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path_to_liar = './raw/liar_dataset/'\n",
        "#counts refer to the speaker's reliability as a whole, interestingly excluding true counts. \n",
        "liar_headers = ('id','label','statement',\n",
        "                'subjects','speaker','job_title',\n",
        "                'state','party','barely_true_counts',\n",
        "                'false_counts','half_true_counts','mostly_true_counts',\n",
        "                'pants_on_fire_counts','context')\n",
        "\n",
        "liar_score = {'true':1,\n",
        "              'mostly-true':1,\n",
        "              'half-true':1,\n",
        "              'barely-true':0,\n",
        "              'false':0,\n",
        "              'pants-fire':0\n",
        "             }"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YEIZsaT3SSFq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Opens dataset, extracts only statement & label and reformats them using formatter()\n",
        "liar_train = pd.read_csv(path_to_liar+'train.tsv',delimiter='\\t',header=None,names=liar_headers)\n",
        "liar_train.loc[:,('statement','label')] = liar_train.loc[:,('statement','label')].apply(formatter,axis=1,result_type='broadcast')\n",
        "liar_test = pd.read_csv(path_to_liar+'test.tsv',delimiter='\\t',header=None,names=liar_headers)\n",
        "liar_test.loc[:,('statement','label')] = liar_test.loc[:,('statement','label')].apply(formatter,axis=1,result_type='broadcast')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BTX4UDOg7ERl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "minlen = 5\n",
        "maxlen = 30\n",
        "liar_train = liar_train[liar_train.statement.apply(lambda x:minlen<=len(x.split())<=maxlen)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ABncuU-7pq-p",
        "colab_type": "text"
      },
      "source": [
        "# Bag of Words + SVM approach"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iN_qrBuX1CSX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#data fed to vectorizer is already cleaned but maybe TODO use their params instead\n",
        "vectorizer = TfidfVectorizer(max_features=200,ngram_range=(1,2)) #gets kwords on principle of very used in doc, but rare across all docs\n",
        "#vectorizer = CountVectorizer(max_features=400) #gets kwords by frequency distribution (is not smart)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7WO9wm6lSSFt",
        "colab_type": "code",
        "outputId": "836d8a4b-1015-4e8c-e49e-d25950b996ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "#trains vectorizer on training dataset, followed by transforming training dataset into vectors\n",
        "train_data = vectorizer.fit_transform(liar_train.statement)\n",
        "x_train = train_data.toarray()\n",
        "y_train = np.array(liar_train.label,dtype=int)\n",
        "#transforms test dataset into vectors without training\n",
        "test_data = vectorizer.transform(liar_test.statement)\n",
        "x_test = test_data.toarray()\n",
        "y_test = np.array(liar_test.label,dtype=int)\n",
        "#see which keywords were extracted as features\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['abortion', 'act', 'actually', 'administration', 'almost', 'america', 'american', 'average', 'back', 'benefit', 'bill', 'billion', 'border', 'budget', 'bush', 'business', 'campaign', 'candidate', 'care', 'child', 'city', 'clinton', 'college', 'come', 'company', 'congress', 'cost', 'could', 'country', 'county', 'court', 'created', 'crime', 'cut', 'debt', 'deficit', 'democrat', 'democratic', 'department', 'district', 'dollar', 'dont', 'economy', 'education', 'election', 'employee', 'even', 'every', 'family', 'federal', 'first', 'five', 'florida', 'four', 'fund', 'funding', 'georgia', 'get', 'give', 'going', 'got', 'gov', 'government', 'governor', 'gun', 'half', 'health', 'health insurance', 'healthcare', 'high', 'higher', 'highest', 'hillary', 'history', 'home', 'house', 'illegal', 'immigrant', 'immigration', 'income', 'increase', 'insurance', 'island', 'job', 'john', 'know', 'last', 'last year', 'law', 'le', 'legislation', 'like', 'lost', 'made', 'majority', 'make', 'many', 'mccain', 'medicare', 'member', 'military', 'million', 'mitt', 'mitt romney', 'money', 'month', 'much', 'nation', 'national', 'nearly', 'never', 'new', 'number', 'obama', 'obamacare', 'obamas', 'office', 'ohio', 'oil', 'one', 'paid', 'party', 'passed', 'past', 'pay', 'people', 'per', 'percent', 'plan', 'president', 'private', 'program', 'proposed', 'public', 'put', 'raise', 'rate', 'record', 'reform', 'republican', 'rhode', 'rick', 'right', 'romney', 'say', 'say obama', 'school', 'scott', 'scott walker', 'security', 'senate', 'since', 'social', 'social security', 'spending', 'spent', 'state', 'still', 'stimulus', 'student', 'support', 'supported', 'system', 'take', 'tax', 'tax cut', 'taxpayer', 'texas', 'thats', 'three', 'time', 'today', 'took', 'top', 'trillion', 'trump', 'two', 'unemployment', 'united', 'united state', 'use', 'used', 'vote', 'voted', 'voter', 'wage', 'walker', 'want', 'war', 'went', 'weve', 'white', 'wisconsin', 'without', 'woman', 'work', 'worker', 'world', 'would', 'year']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Rirubr1SSF0",
        "colab_type": "code",
        "outputId": "e78b624e-6ba6-4334-e464-a6ad9ab7b327",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "#fits SVM on vectorized training dataset\n",
        "clf = SVC(kernel='linear')\n",
        "clf.fit(x_train,y_train)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
              "    decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
              "    kernel='linear', max_iter=-1, probability=False, random_state=None,\n",
              "    shrinking=True, tol=0.001, verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qJ9Q0csBSSF-",
        "colab_type": "code",
        "outputId": "7929498d-2c95-4cc1-b35d-1e24b1250ad1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y_pred = clf.predict(x_test)\n",
        "printScore(y_pred,y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 60.1%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tlaGAKL33NFG",
        "colab_type": "text"
      },
      "source": [
        "## Analysis\n",
        "Bag of Words + SVM is essentially classifying fake news by similarity to other fake news via keywords. Many of these keywords picked by tfidf vectorizer are names and terms that are often engaged with controversial topics, likely due to the political nature of the dataset. Likewise, since fake news tends to surround controversial topics, controversial keywords end up being the main features used by SVM to classify even though they don't entail being fake. Unfortunately, lemmatization doesn't seem to fix things like \"theyre\" or \"cuban\" vs \"cuba\"."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TKCvTuIeq_8m",
        "colab_type": "text"
      },
      "source": [
        "# ELMo Sum + SVM approach"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5eCoQebS8rxZ",
        "colab_type": "text"
      },
      "source": [
        "!pip install allennlp"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASfwlsEmSSFO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from allennlp.modules.elmo import Elmo, batch_to_ids\n",
        "from torch import nn\n",
        "from sklearn.decomposition import PCA\n",
        "options_file = \"https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json\"\n",
        "weight_file = \"https://allennlp.s3.amazonaws.com/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5\"\n",
        "elmo = Elmo(options_file, weight_file, 1, dropout=0).to('cuda') #1 representation only"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fxAR9wCB6tTw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 32\n",
        "def getEmbedded(df):\n",
        "    batches = np.array_split(df,int(len(df)/batch_size))\n",
        "    edf = pd.DataFrame({})\n",
        "    for batch in batches:\n",
        "        sentences = batch.statement.str.split()\n",
        "        character_ids = batch_to_ids(sentences).to('cuda')\n",
        "        embeddings =  elmo(character_ids)['elmo_representations'] #[representation][sent_no][word_no (padded)]\n",
        "        normalized = nn.functional.softmax(embeddings[0].sum(1),dim=1).to('cpu').detach()\n",
        "        embed = pd.DataFrame(normalized.numpy())\n",
        "        embed = embed.assign(label=batch.label.values)\n",
        "        edf = edf.append(embed,ignore_index=True)\n",
        "    return edf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WUEtTSxZSSF3",
        "colab_type": "code",
        "outputId": "2a6e99e4-0381-4f4c-991a-922a164f24a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 609
        }
      },
      "source": [
        "train_df = getEmbedded(liar_train)\n",
        "test_df = getEmbedded(liar_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>...</th>\n",
              "      <th>985</th>\n",
              "      <th>986</th>\n",
              "      <th>987</th>\n",
              "      <th>988</th>\n",
              "      <th>989</th>\n",
              "      <th>990</th>\n",
              "      <th>991</th>\n",
              "      <th>992</th>\n",
              "      <th>993</th>\n",
              "      <th>994</th>\n",
              "      <th>995</th>\n",
              "      <th>996</th>\n",
              "      <th>997</th>\n",
              "      <th>998</th>\n",
              "      <th>999</th>\n",
              "      <th>1000</th>\n",
              "      <th>1001</th>\n",
              "      <th>1002</th>\n",
              "      <th>1003</th>\n",
              "      <th>1004</th>\n",
              "      <th>1005</th>\n",
              "      <th>1006</th>\n",
              "      <th>1007</th>\n",
              "      <th>1008</th>\n",
              "      <th>1009</th>\n",
              "      <th>1010</th>\n",
              "      <th>1011</th>\n",
              "      <th>1012</th>\n",
              "      <th>1013</th>\n",
              "      <th>1014</th>\n",
              "      <th>1015</th>\n",
              "      <th>1016</th>\n",
              "      <th>1017</th>\n",
              "      <th>1018</th>\n",
              "      <th>1019</th>\n",
              "      <th>1020</th>\n",
              "      <th>1021</th>\n",
              "      <th>1022</th>\n",
              "      <th>1023</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3.575464e-07</td>\n",
              "      <td>3.161658e-07</td>\n",
              "      <td>5.078559e-08</td>\n",
              "      <td>2.582136e-08</td>\n",
              "      <td>9.453494e-08</td>\n",
              "      <td>1.321333e-05</td>\n",
              "      <td>2.250624e-08</td>\n",
              "      <td>8.324159e-07</td>\n",
              "      <td>8.880279e-08</td>\n",
              "      <td>1.889627e-07</td>\n",
              "      <td>4.245820e-08</td>\n",
              "      <td>6.092341e-06</td>\n",
              "      <td>2.380327e-08</td>\n",
              "      <td>2.220460e-07</td>\n",
              "      <td>6.266293e-08</td>\n",
              "      <td>2.327942e-08</td>\n",
              "      <td>3.523541e-07</td>\n",
              "      <td>6.563824e-08</td>\n",
              "      <td>7.673962e-08</td>\n",
              "      <td>1.422934e-08</td>\n",
              "      <td>1.361767e-06</td>\n",
              "      <td>2.284145e-07</td>\n",
              "      <td>3.371377e-08</td>\n",
              "      <td>1.785008e-08</td>\n",
              "      <td>7.911170e-07</td>\n",
              "      <td>5.995866e-07</td>\n",
              "      <td>9.646606e-08</td>\n",
              "      <td>8.656997e-08</td>\n",
              "      <td>6.128693e-08</td>\n",
              "      <td>4.203974e-08</td>\n",
              "      <td>5.419511e-07</td>\n",
              "      <td>1.006815e-07</td>\n",
              "      <td>7.163814e-08</td>\n",
              "      <td>3.863121e-08</td>\n",
              "      <td>2.674054e-08</td>\n",
              "      <td>5.975621e-08</td>\n",
              "      <td>1.014583e-07</td>\n",
              "      <td>5.718601e-07</td>\n",
              "      <td>9.050759e-09</td>\n",
              "      <td>1.363908e-06</td>\n",
              "      <td>...</td>\n",
              "      <td>6.399458e-08</td>\n",
              "      <td>1.192406e-05</td>\n",
              "      <td>1.726750e-07</td>\n",
              "      <td>4.245786e-07</td>\n",
              "      <td>7.344992e-09</td>\n",
              "      <td>9.518085e-08</td>\n",
              "      <td>3.721983e-06</td>\n",
              "      <td>1.655280e-07</td>\n",
              "      <td>7.924373e-08</td>\n",
              "      <td>1.346388e-08</td>\n",
              "      <td>2.316122e-08</td>\n",
              "      <td>3.365276e-07</td>\n",
              "      <td>7.175941e-09</td>\n",
              "      <td>2.341029e-08</td>\n",
              "      <td>2.931698e-07</td>\n",
              "      <td>4.458563e-07</td>\n",
              "      <td>4.940963e-07</td>\n",
              "      <td>5.056665e-08</td>\n",
              "      <td>1.374802e-09</td>\n",
              "      <td>3.273360e-07</td>\n",
              "      <td>3.663583e-08</td>\n",
              "      <td>4.873589e-08</td>\n",
              "      <td>1.310111e-08</td>\n",
              "      <td>3.499698e-08</td>\n",
              "      <td>1.246058e-07</td>\n",
              "      <td>3.984230e-08</td>\n",
              "      <td>4.606122e-08</td>\n",
              "      <td>1.491928e-07</td>\n",
              "      <td>3.323657e-08</td>\n",
              "      <td>1.932655e-07</td>\n",
              "      <td>3.734891e-07</td>\n",
              "      <td>6.034442e-06</td>\n",
              "      <td>2.155405e-06</td>\n",
              "      <td>7.129804e-08</td>\n",
              "      <td>2.934389e-07</td>\n",
              "      <td>2.287612e-07</td>\n",
              "      <td>9.990690e-08</td>\n",
              "      <td>1.429624e-08</td>\n",
              "      <td>3.311136e-06</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.599470e-07</td>\n",
              "      <td>3.906797e-11</td>\n",
              "      <td>2.611124e-08</td>\n",
              "      <td>4.064196e-10</td>\n",
              "      <td>4.452543e-09</td>\n",
              "      <td>1.559676e-11</td>\n",
              "      <td>4.972526e-09</td>\n",
              "      <td>1.128676e-08</td>\n",
              "      <td>6.200806e-08</td>\n",
              "      <td>2.412147e-11</td>\n",
              "      <td>1.394036e-10</td>\n",
              "      <td>1.329038e-07</td>\n",
              "      <td>5.137624e-10</td>\n",
              "      <td>8.279552e-09</td>\n",
              "      <td>1.188420e-10</td>\n",
              "      <td>9.656483e-10</td>\n",
              "      <td>8.200171e-09</td>\n",
              "      <td>1.910349e-07</td>\n",
              "      <td>2.513693e-10</td>\n",
              "      <td>8.487887e-12</td>\n",
              "      <td>5.074328e-08</td>\n",
              "      <td>6.328862e-08</td>\n",
              "      <td>3.335613e-10</td>\n",
              "      <td>3.125939e-09</td>\n",
              "      <td>5.824008e-06</td>\n",
              "      <td>5.274549e-10</td>\n",
              "      <td>1.272596e-09</td>\n",
              "      <td>3.108153e-11</td>\n",
              "      <td>1.872244e-11</td>\n",
              "      <td>2.755560e-09</td>\n",
              "      <td>1.352122e-11</td>\n",
              "      <td>1.971132e-10</td>\n",
              "      <td>2.678495e-08</td>\n",
              "      <td>1.448407e-13</td>\n",
              "      <td>2.235946e-10</td>\n",
              "      <td>2.789895e-08</td>\n",
              "      <td>4.235265e-12</td>\n",
              "      <td>6.888509e-11</td>\n",
              "      <td>3.043412e-11</td>\n",
              "      <td>6.588111e-07</td>\n",
              "      <td>...</td>\n",
              "      <td>2.626303e-08</td>\n",
              "      <td>2.340297e-08</td>\n",
              "      <td>2.103257e-08</td>\n",
              "      <td>1.307327e-09</td>\n",
              "      <td>1.417098e-11</td>\n",
              "      <td>1.249224e-07</td>\n",
              "      <td>4.277743e-08</td>\n",
              "      <td>3.850852e-11</td>\n",
              "      <td>2.678787e-11</td>\n",
              "      <td>1.204867e-09</td>\n",
              "      <td>8.096161e-11</td>\n",
              "      <td>9.343012e-10</td>\n",
              "      <td>7.310395e-11</td>\n",
              "      <td>6.644527e-09</td>\n",
              "      <td>1.110292e-08</td>\n",
              "      <td>7.676096e-09</td>\n",
              "      <td>6.318763e-09</td>\n",
              "      <td>4.333890e-07</td>\n",
              "      <td>3.187895e-10</td>\n",
              "      <td>1.132950e-09</td>\n",
              "      <td>7.599781e-11</td>\n",
              "      <td>6.993562e-10</td>\n",
              "      <td>4.900661e-11</td>\n",
              "      <td>2.321422e-10</td>\n",
              "      <td>2.172521e-10</td>\n",
              "      <td>4.990607e-09</td>\n",
              "      <td>3.726733e-11</td>\n",
              "      <td>2.938928e-10</td>\n",
              "      <td>8.118273e-11</td>\n",
              "      <td>4.764495e-10</td>\n",
              "      <td>5.354232e-09</td>\n",
              "      <td>1.404089e-08</td>\n",
              "      <td>8.922941e-09</td>\n",
              "      <td>1.483065e-08</td>\n",
              "      <td>1.883586e-09</td>\n",
              "      <td>6.913089e-13</td>\n",
              "      <td>5.489281e-09</td>\n",
              "      <td>6.146229e-08</td>\n",
              "      <td>1.360501e-08</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>9.814446e-12</td>\n",
              "      <td>5.079448e-11</td>\n",
              "      <td>6.235529e-11</td>\n",
              "      <td>8.781818e-12</td>\n",
              "      <td>1.523425e-08</td>\n",
              "      <td>5.529006e-09</td>\n",
              "      <td>3.581679e-11</td>\n",
              "      <td>1.797976e-09</td>\n",
              "      <td>2.404363e-12</td>\n",
              "      <td>3.565200e-10</td>\n",
              "      <td>1.625956e-10</td>\n",
              "      <td>1.388733e-09</td>\n",
              "      <td>1.279130e-14</td>\n",
              "      <td>2.570001e-09</td>\n",
              "      <td>7.621947e-13</td>\n",
              "      <td>9.070885e-09</td>\n",
              "      <td>8.670822e-08</td>\n",
              "      <td>4.177335e-09</td>\n",
              "      <td>1.613342e-11</td>\n",
              "      <td>5.425477e-10</td>\n",
              "      <td>1.173229e-07</td>\n",
              "      <td>6.194594e-12</td>\n",
              "      <td>2.442850e-12</td>\n",
              "      <td>2.046868e-11</td>\n",
              "      <td>2.615798e-05</td>\n",
              "      <td>1.569983e-09</td>\n",
              "      <td>4.316894e-11</td>\n",
              "      <td>9.201715e-10</td>\n",
              "      <td>4.459270e-09</td>\n",
              "      <td>1.570928e-07</td>\n",
              "      <td>3.530238e-09</td>\n",
              "      <td>3.392414e-09</td>\n",
              "      <td>1.478514e-07</td>\n",
              "      <td>5.315302e-11</td>\n",
              "      <td>9.314409e-10</td>\n",
              "      <td>1.319113e-10</td>\n",
              "      <td>8.159178e-09</td>\n",
              "      <td>8.325095e-11</td>\n",
              "      <td>5.756414e-08</td>\n",
              "      <td>2.713639e-07</td>\n",
              "      <td>...</td>\n",
              "      <td>2.638092e-09</td>\n",
              "      <td>1.128282e-06</td>\n",
              "      <td>2.782590e-08</td>\n",
              "      <td>3.406920e-09</td>\n",
              "      <td>8.423344e-14</td>\n",
              "      <td>3.819630e-10</td>\n",
              "      <td>2.047126e-07</td>\n",
              "      <td>8.704749e-10</td>\n",
              "      <td>4.329496e-10</td>\n",
              "      <td>5.441815e-12</td>\n",
              "      <td>3.985891e-12</td>\n",
              "      <td>2.106085e-09</td>\n",
              "      <td>9.646708e-12</td>\n",
              "      <td>2.574482e-11</td>\n",
              "      <td>1.641902e-08</td>\n",
              "      <td>3.397954e-11</td>\n",
              "      <td>3.187865e-08</td>\n",
              "      <td>2.727566e-09</td>\n",
              "      <td>2.866984e-12</td>\n",
              "      <td>4.230520e-09</td>\n",
              "      <td>2.377531e-11</td>\n",
              "      <td>1.252843e-09</td>\n",
              "      <td>2.054892e-10</td>\n",
              "      <td>2.970592e-10</td>\n",
              "      <td>2.413894e-11</td>\n",
              "      <td>1.682336e-09</td>\n",
              "      <td>2.736865e-12</td>\n",
              "      <td>1.566578e-10</td>\n",
              "      <td>9.644997e-12</td>\n",
              "      <td>2.777163e-10</td>\n",
              "      <td>6.348471e-10</td>\n",
              "      <td>6.057572e-09</td>\n",
              "      <td>2.243938e-10</td>\n",
              "      <td>2.039026e-07</td>\n",
              "      <td>1.918022e-09</td>\n",
              "      <td>7.359054e-12</td>\n",
              "      <td>2.286350e-10</td>\n",
              "      <td>2.364420e-08</td>\n",
              "      <td>1.850572e-10</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2.105866e-08</td>\n",
              "      <td>4.769444e-08</td>\n",
              "      <td>6.389127e-08</td>\n",
              "      <td>9.991250e-10</td>\n",
              "      <td>5.190867e-08</td>\n",
              "      <td>1.804813e-07</td>\n",
              "      <td>3.957941e-07</td>\n",
              "      <td>8.896820e-07</td>\n",
              "      <td>5.276281e-09</td>\n",
              "      <td>5.607099e-07</td>\n",
              "      <td>5.218061e-10</td>\n",
              "      <td>1.756783e-09</td>\n",
              "      <td>1.089420e-09</td>\n",
              "      <td>1.321725e-06</td>\n",
              "      <td>9.052951e-09</td>\n",
              "      <td>6.051816e-08</td>\n",
              "      <td>1.539918e-08</td>\n",
              "      <td>5.993585e-05</td>\n",
              "      <td>2.339029e-08</td>\n",
              "      <td>3.270765e-10</td>\n",
              "      <td>1.373959e-07</td>\n",
              "      <td>3.652406e-06</td>\n",
              "      <td>1.009335e-08</td>\n",
              "      <td>2.073680e-08</td>\n",
              "      <td>2.661620e-06</td>\n",
              "      <td>8.050912e-08</td>\n",
              "      <td>1.688675e-07</td>\n",
              "      <td>8.782989e-10</td>\n",
              "      <td>1.702393e-07</td>\n",
              "      <td>9.422458e-08</td>\n",
              "      <td>1.410757e-08</td>\n",
              "      <td>1.834457e-09</td>\n",
              "      <td>1.690430e-08</td>\n",
              "      <td>1.087439e-11</td>\n",
              "      <td>1.065383e-08</td>\n",
              "      <td>3.130725e-08</td>\n",
              "      <td>4.235772e-07</td>\n",
              "      <td>1.581905e-07</td>\n",
              "      <td>3.870149e-08</td>\n",
              "      <td>5.947024e-08</td>\n",
              "      <td>...</td>\n",
              "      <td>1.202307e-07</td>\n",
              "      <td>9.834765e-06</td>\n",
              "      <td>4.092449e-09</td>\n",
              "      <td>1.850667e-06</td>\n",
              "      <td>5.476257e-10</td>\n",
              "      <td>9.978153e-09</td>\n",
              "      <td>1.707854e-07</td>\n",
              "      <td>1.212246e-08</td>\n",
              "      <td>5.787810e-10</td>\n",
              "      <td>5.793833e-09</td>\n",
              "      <td>7.759588e-09</td>\n",
              "      <td>3.076003e-08</td>\n",
              "      <td>5.138128e-08</td>\n",
              "      <td>2.395287e-05</td>\n",
              "      <td>1.693310e-07</td>\n",
              "      <td>4.081675e-09</td>\n",
              "      <td>2.241764e-08</td>\n",
              "      <td>5.224475e-06</td>\n",
              "      <td>6.885739e-10</td>\n",
              "      <td>2.565301e-09</td>\n",
              "      <td>9.902637e-07</td>\n",
              "      <td>1.561101e-08</td>\n",
              "      <td>1.365882e-09</td>\n",
              "      <td>2.433853e-08</td>\n",
              "      <td>1.422183e-07</td>\n",
              "      <td>1.363129e-08</td>\n",
              "      <td>7.572675e-08</td>\n",
              "      <td>8.134992e-08</td>\n",
              "      <td>6.114144e-08</td>\n",
              "      <td>3.491663e-08</td>\n",
              "      <td>1.266268e-07</td>\n",
              "      <td>1.581471e-06</td>\n",
              "      <td>1.900756e-06</td>\n",
              "      <td>1.496353e-08</td>\n",
              "      <td>1.158873e-08</td>\n",
              "      <td>4.220900e-07</td>\n",
              "      <td>3.535834e-07</td>\n",
              "      <td>1.492028e-07</td>\n",
              "      <td>1.000970e-08</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.778951e-04</td>\n",
              "      <td>1.766992e-04</td>\n",
              "      <td>5.124250e-04</td>\n",
              "      <td>8.568383e-05</td>\n",
              "      <td>5.201604e-04</td>\n",
              "      <td>3.373282e-04</td>\n",
              "      <td>5.317577e-05</td>\n",
              "      <td>2.548842e-04</td>\n",
              "      <td>4.712872e-04</td>\n",
              "      <td>8.512393e-06</td>\n",
              "      <td>3.231085e-06</td>\n",
              "      <td>1.407595e-04</td>\n",
              "      <td>5.428588e-05</td>\n",
              "      <td>3.990656e-04</td>\n",
              "      <td>1.030789e-04</td>\n",
              "      <td>6.778256e-04</td>\n",
              "      <td>1.078440e-04</td>\n",
              "      <td>1.437863e-04</td>\n",
              "      <td>8.801126e-06</td>\n",
              "      <td>8.791160e-06</td>\n",
              "      <td>1.993705e-04</td>\n",
              "      <td>6.988014e-04</td>\n",
              "      <td>6.359768e-05</td>\n",
              "      <td>2.893856e-05</td>\n",
              "      <td>1.202031e-03</td>\n",
              "      <td>2.477498e-04</td>\n",
              "      <td>3.412782e-04</td>\n",
              "      <td>2.703595e-04</td>\n",
              "      <td>3.645394e-04</td>\n",
              "      <td>4.708523e-04</td>\n",
              "      <td>5.604607e-05</td>\n",
              "      <td>7.972860e-05</td>\n",
              "      <td>2.594352e-05</td>\n",
              "      <td>8.246379e-06</td>\n",
              "      <td>1.461248e-04</td>\n",
              "      <td>1.626034e-03</td>\n",
              "      <td>2.477213e-05</td>\n",
              "      <td>2.614605e-05</td>\n",
              "      <td>3.285197e-04</td>\n",
              "      <td>6.846720e-04</td>\n",
              "      <td>...</td>\n",
              "      <td>1.205156e-04</td>\n",
              "      <td>3.363315e-04</td>\n",
              "      <td>8.663203e-05</td>\n",
              "      <td>5.381540e-04</td>\n",
              "      <td>1.165234e-05</td>\n",
              "      <td>1.141869e-04</td>\n",
              "      <td>2.382680e-04</td>\n",
              "      <td>1.422321e-04</td>\n",
              "      <td>4.889577e-05</td>\n",
              "      <td>6.836492e-05</td>\n",
              "      <td>5.405393e-05</td>\n",
              "      <td>4.203666e-05</td>\n",
              "      <td>2.781126e-05</td>\n",
              "      <td>9.481876e-05</td>\n",
              "      <td>4.552653e-05</td>\n",
              "      <td>8.898852e-04</td>\n",
              "      <td>9.032280e-04</td>\n",
              "      <td>2.732727e-04</td>\n",
              "      <td>4.811449e-05</td>\n",
              "      <td>6.766611e-06</td>\n",
              "      <td>6.360335e-06</td>\n",
              "      <td>1.539866e-04</td>\n",
              "      <td>1.987431e-05</td>\n",
              "      <td>2.927668e-04</td>\n",
              "      <td>5.605511e-05</td>\n",
              "      <td>2.192786e-05</td>\n",
              "      <td>4.856278e-06</td>\n",
              "      <td>2.620869e-05</td>\n",
              "      <td>3.054487e-05</td>\n",
              "      <td>1.553552e-04</td>\n",
              "      <td>3.281241e-04</td>\n",
              "      <td>8.823006e-05</td>\n",
              "      <td>3.894698e-04</td>\n",
              "      <td>7.493081e-05</td>\n",
              "      <td>4.051361e-05</td>\n",
              "      <td>5.830277e-05</td>\n",
              "      <td>2.714839e-05</td>\n",
              "      <td>3.033737e-04</td>\n",
              "      <td>7.318950e-04</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9707</th>\n",
              "      <td>3.766673e-04</td>\n",
              "      <td>1.946348e-07</td>\n",
              "      <td>7.051593e-06</td>\n",
              "      <td>3.626067e-06</td>\n",
              "      <td>1.081029e-05</td>\n",
              "      <td>3.251797e-04</td>\n",
              "      <td>2.726597e-06</td>\n",
              "      <td>1.325924e-05</td>\n",
              "      <td>9.265475e-06</td>\n",
              "      <td>1.042913e-06</td>\n",
              "      <td>3.057261e-07</td>\n",
              "      <td>1.826487e-05</td>\n",
              "      <td>1.168401e-08</td>\n",
              "      <td>1.393778e-05</td>\n",
              "      <td>1.789972e-06</td>\n",
              "      <td>5.431098e-06</td>\n",
              "      <td>6.450532e-05</td>\n",
              "      <td>9.094427e-07</td>\n",
              "      <td>1.206856e-06</td>\n",
              "      <td>1.155524e-07</td>\n",
              "      <td>7.416581e-06</td>\n",
              "      <td>4.095186e-05</td>\n",
              "      <td>1.596095e-05</td>\n",
              "      <td>1.491532e-06</td>\n",
              "      <td>1.145161e-04</td>\n",
              "      <td>1.932130e-05</td>\n",
              "      <td>8.245692e-05</td>\n",
              "      <td>1.045530e-06</td>\n",
              "      <td>1.597032e-06</td>\n",
              "      <td>3.306659e-07</td>\n",
              "      <td>4.014326e-06</td>\n",
              "      <td>8.449621e-06</td>\n",
              "      <td>1.160289e-06</td>\n",
              "      <td>2.151161e-07</td>\n",
              "      <td>4.654628e-07</td>\n",
              "      <td>1.526923e-06</td>\n",
              "      <td>7.133885e-08</td>\n",
              "      <td>4.199758e-06</td>\n",
              "      <td>8.185730e-07</td>\n",
              "      <td>4.193466e-04</td>\n",
              "      <td>...</td>\n",
              "      <td>1.976497e-05</td>\n",
              "      <td>3.271068e-04</td>\n",
              "      <td>4.043874e-06</td>\n",
              "      <td>1.153946e-04</td>\n",
              "      <td>2.202389e-06</td>\n",
              "      <td>2.332841e-04</td>\n",
              "      <td>2.820306e-05</td>\n",
              "      <td>2.033407e-06</td>\n",
              "      <td>7.778578e-07</td>\n",
              "      <td>1.052410e-06</td>\n",
              "      <td>3.528736e-07</td>\n",
              "      <td>1.599803e-05</td>\n",
              "      <td>2.899429e-06</td>\n",
              "      <td>3.010037e-06</td>\n",
              "      <td>7.689388e-06</td>\n",
              "      <td>2.605994e-07</td>\n",
              "      <td>1.439389e-05</td>\n",
              "      <td>1.040470e-05</td>\n",
              "      <td>2.378560e-07</td>\n",
              "      <td>3.635931e-05</td>\n",
              "      <td>1.163204e-06</td>\n",
              "      <td>2.708984e-07</td>\n",
              "      <td>2.379207e-07</td>\n",
              "      <td>7.838115e-07</td>\n",
              "      <td>1.634663e-06</td>\n",
              "      <td>3.634861e-06</td>\n",
              "      <td>7.375806e-07</td>\n",
              "      <td>7.640807e-06</td>\n",
              "      <td>2.584081e-06</td>\n",
              "      <td>2.292896e-08</td>\n",
              "      <td>1.114224e-06</td>\n",
              "      <td>6.959592e-05</td>\n",
              "      <td>3.267319e-05</td>\n",
              "      <td>6.269131e-06</td>\n",
              "      <td>2.705736e-07</td>\n",
              "      <td>6.210498e-08</td>\n",
              "      <td>1.059216e-05</td>\n",
              "      <td>1.339493e-05</td>\n",
              "      <td>9.895754e-07</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9708</th>\n",
              "      <td>2.311359e-05</td>\n",
              "      <td>2.485752e-07</td>\n",
              "      <td>9.899273e-06</td>\n",
              "      <td>1.035974e-06</td>\n",
              "      <td>1.541959e-06</td>\n",
              "      <td>7.924552e-05</td>\n",
              "      <td>1.188665e-06</td>\n",
              "      <td>4.390112e-06</td>\n",
              "      <td>1.921303e-06</td>\n",
              "      <td>7.867956e-08</td>\n",
              "      <td>4.069365e-08</td>\n",
              "      <td>1.147670e-04</td>\n",
              "      <td>5.757890e-08</td>\n",
              "      <td>6.466606e-06</td>\n",
              "      <td>7.825916e-08</td>\n",
              "      <td>1.024555e-05</td>\n",
              "      <td>3.470148e-05</td>\n",
              "      <td>3.305501e-06</td>\n",
              "      <td>3.930148e-06</td>\n",
              "      <td>8.402445e-06</td>\n",
              "      <td>4.251002e-04</td>\n",
              "      <td>3.239555e-06</td>\n",
              "      <td>3.291868e-07</td>\n",
              "      <td>4.990103e-07</td>\n",
              "      <td>2.059397e-04</td>\n",
              "      <td>4.476612e-07</td>\n",
              "      <td>9.136708e-06</td>\n",
              "      <td>1.370978e-06</td>\n",
              "      <td>1.051606e-04</td>\n",
              "      <td>1.990459e-05</td>\n",
              "      <td>5.762588e-06</td>\n",
              "      <td>2.103512e-06</td>\n",
              "      <td>4.841814e-06</td>\n",
              "      <td>1.291834e-07</td>\n",
              "      <td>1.683191e-06</td>\n",
              "      <td>2.304990e-06</td>\n",
              "      <td>1.739236e-05</td>\n",
              "      <td>1.269168e-05</td>\n",
              "      <td>9.761863e-06</td>\n",
              "      <td>1.298137e-04</td>\n",
              "      <td>...</td>\n",
              "      <td>5.530300e-06</td>\n",
              "      <td>4.279315e-05</td>\n",
              "      <td>2.071499e-06</td>\n",
              "      <td>1.796400e-05</td>\n",
              "      <td>5.361258e-07</td>\n",
              "      <td>6.398516e-05</td>\n",
              "      <td>1.155972e-06</td>\n",
              "      <td>2.509023e-06</td>\n",
              "      <td>5.693133e-06</td>\n",
              "      <td>1.468711e-06</td>\n",
              "      <td>5.785572e-06</td>\n",
              "      <td>2.980505e-05</td>\n",
              "      <td>8.410626e-07</td>\n",
              "      <td>4.784881e-08</td>\n",
              "      <td>1.720037e-04</td>\n",
              "      <td>1.726609e-05</td>\n",
              "      <td>8.530221e-05</td>\n",
              "      <td>5.707533e-06</td>\n",
              "      <td>4.682849e-07</td>\n",
              "      <td>3.304749e-05</td>\n",
              "      <td>6.096428e-06</td>\n",
              "      <td>3.467662e-06</td>\n",
              "      <td>4.476191e-06</td>\n",
              "      <td>6.466782e-07</td>\n",
              "      <td>3.694591e-07</td>\n",
              "      <td>8.192866e-06</td>\n",
              "      <td>2.130341e-07</td>\n",
              "      <td>4.910041e-07</td>\n",
              "      <td>2.810497e-06</td>\n",
              "      <td>1.424297e-06</td>\n",
              "      <td>1.641425e-05</td>\n",
              "      <td>3.854972e-05</td>\n",
              "      <td>2.242332e-05</td>\n",
              "      <td>4.683494e-06</td>\n",
              "      <td>8.658430e-07</td>\n",
              "      <td>1.789471e-07</td>\n",
              "      <td>7.930410e-07</td>\n",
              "      <td>1.130424e-05</td>\n",
              "      <td>9.300837e-06</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9709</th>\n",
              "      <td>2.240841e-07</td>\n",
              "      <td>9.109556e-10</td>\n",
              "      <td>7.405200e-13</td>\n",
              "      <td>5.907319e-13</td>\n",
              "      <td>3.957532e-09</td>\n",
              "      <td>4.279110e-10</td>\n",
              "      <td>4.348365e-12</td>\n",
              "      <td>8.668226e-10</td>\n",
              "      <td>1.380913e-09</td>\n",
              "      <td>6.751173e-09</td>\n",
              "      <td>1.099995e-15</td>\n",
              "      <td>8.148381e-12</td>\n",
              "      <td>9.215740e-14</td>\n",
              "      <td>4.693584e-13</td>\n",
              "      <td>2.728490e-13</td>\n",
              "      <td>1.244993e-09</td>\n",
              "      <td>3.538933e-12</td>\n",
              "      <td>1.123793e-08</td>\n",
              "      <td>3.234205e-12</td>\n",
              "      <td>7.820219e-15</td>\n",
              "      <td>5.276064e-08</td>\n",
              "      <td>1.104225e-10</td>\n",
              "      <td>2.018426e-10</td>\n",
              "      <td>4.297834e-13</td>\n",
              "      <td>6.598664e-07</td>\n",
              "      <td>9.020899e-12</td>\n",
              "      <td>2.216860e-09</td>\n",
              "      <td>7.288263e-10</td>\n",
              "      <td>1.350794e-12</td>\n",
              "      <td>1.749235e-11</td>\n",
              "      <td>2.454151e-11</td>\n",
              "      <td>5.913094e-12</td>\n",
              "      <td>1.390038e-13</td>\n",
              "      <td>1.132427e-14</td>\n",
              "      <td>1.061896e-12</td>\n",
              "      <td>5.194219e-11</td>\n",
              "      <td>1.202106e-12</td>\n",
              "      <td>1.098367e-10</td>\n",
              "      <td>1.217117e-12</td>\n",
              "      <td>4.496731e-11</td>\n",
              "      <td>...</td>\n",
              "      <td>1.324916e-10</td>\n",
              "      <td>2.371454e-08</td>\n",
              "      <td>2.683575e-11</td>\n",
              "      <td>1.377795e-09</td>\n",
              "      <td>1.426936e-15</td>\n",
              "      <td>2.396871e-11</td>\n",
              "      <td>1.049699e-10</td>\n",
              "      <td>8.522845e-13</td>\n",
              "      <td>6.106567e-13</td>\n",
              "      <td>4.420491e-12</td>\n",
              "      <td>3.134625e-14</td>\n",
              "      <td>5.974901e-13</td>\n",
              "      <td>5.753752e-12</td>\n",
              "      <td>1.556318e-12</td>\n",
              "      <td>8.924250e-10</td>\n",
              "      <td>1.511630e-11</td>\n",
              "      <td>2.974334e-08</td>\n",
              "      <td>2.596519e-11</td>\n",
              "      <td>3.785194e-13</td>\n",
              "      <td>7.513243e-12</td>\n",
              "      <td>1.622556e-11</td>\n",
              "      <td>1.294736e-10</td>\n",
              "      <td>2.837970e-14</td>\n",
              "      <td>1.762958e-11</td>\n",
              "      <td>2.621592e-13</td>\n",
              "      <td>4.531729e-12</td>\n",
              "      <td>3.620962e-13</td>\n",
              "      <td>8.440012e-11</td>\n",
              "      <td>1.711253e-14</td>\n",
              "      <td>2.211960e-13</td>\n",
              "      <td>3.965194e-12</td>\n",
              "      <td>4.383436e-10</td>\n",
              "      <td>1.823601e-10</td>\n",
              "      <td>1.117017e-11</td>\n",
              "      <td>3.330761e-11</td>\n",
              "      <td>1.008081e-12</td>\n",
              "      <td>1.807621e-12</td>\n",
              "      <td>5.977589e-10</td>\n",
              "      <td>1.320440e-11</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9710</th>\n",
              "      <td>6.114797e-06</td>\n",
              "      <td>7.439993e-06</td>\n",
              "      <td>1.323335e-05</td>\n",
              "      <td>4.895729e-06</td>\n",
              "      <td>1.739657e-05</td>\n",
              "      <td>1.247517e-05</td>\n",
              "      <td>1.956058e-05</td>\n",
              "      <td>3.480841e-06</td>\n",
              "      <td>4.142369e-05</td>\n",
              "      <td>9.252522e-06</td>\n",
              "      <td>3.960516e-07</td>\n",
              "      <td>2.408232e-05</td>\n",
              "      <td>1.543257e-06</td>\n",
              "      <td>7.159808e-05</td>\n",
              "      <td>2.281814e-04</td>\n",
              "      <td>4.687876e-06</td>\n",
              "      <td>1.100158e-05</td>\n",
              "      <td>6.346097e-06</td>\n",
              "      <td>7.495958e-06</td>\n",
              "      <td>1.161165e-05</td>\n",
              "      <td>8.014476e-05</td>\n",
              "      <td>3.629446e-05</td>\n",
              "      <td>7.613362e-06</td>\n",
              "      <td>3.580020e-06</td>\n",
              "      <td>9.802068e-06</td>\n",
              "      <td>1.321056e-05</td>\n",
              "      <td>4.265251e-06</td>\n",
              "      <td>2.008242e-06</td>\n",
              "      <td>4.146380e-05</td>\n",
              "      <td>2.707765e-05</td>\n",
              "      <td>4.584047e-05</td>\n",
              "      <td>5.467243e-05</td>\n",
              "      <td>8.410949e-06</td>\n",
              "      <td>1.272498e-07</td>\n",
              "      <td>1.870545e-06</td>\n",
              "      <td>4.720008e-05</td>\n",
              "      <td>1.422714e-06</td>\n",
              "      <td>4.287850e-04</td>\n",
              "      <td>1.610026e-06</td>\n",
              "      <td>1.312093e-05</td>\n",
              "      <td>...</td>\n",
              "      <td>3.101019e-06</td>\n",
              "      <td>1.092764e-04</td>\n",
              "      <td>9.922338e-05</td>\n",
              "      <td>3.748980e-05</td>\n",
              "      <td>2.474134e-07</td>\n",
              "      <td>2.757185e-06</td>\n",
              "      <td>9.547790e-05</td>\n",
              "      <td>2.978139e-06</td>\n",
              "      <td>8.291810e-06</td>\n",
              "      <td>1.940731e-06</td>\n",
              "      <td>7.559344e-07</td>\n",
              "      <td>9.534700e-06</td>\n",
              "      <td>2.990727e-05</td>\n",
              "      <td>1.002952e-04</td>\n",
              "      <td>2.765035e-06</td>\n",
              "      <td>1.617320e-05</td>\n",
              "      <td>2.619814e-05</td>\n",
              "      <td>8.604498e-06</td>\n",
              "      <td>4.864629e-07</td>\n",
              "      <td>1.158140e-05</td>\n",
              "      <td>1.235739e-05</td>\n",
              "      <td>2.037811e-05</td>\n",
              "      <td>2.669052e-06</td>\n",
              "      <td>2.411492e-06</td>\n",
              "      <td>2.321547e-05</td>\n",
              "      <td>1.024973e-05</td>\n",
              "      <td>1.251164e-05</td>\n",
              "      <td>2.305570e-06</td>\n",
              "      <td>2.162164e-05</td>\n",
              "      <td>5.339029e-06</td>\n",
              "      <td>2.865189e-05</td>\n",
              "      <td>5.209497e-05</td>\n",
              "      <td>7.428685e-06</td>\n",
              "      <td>5.252917e-06</td>\n",
              "      <td>2.948576e-06</td>\n",
              "      <td>2.144192e-06</td>\n",
              "      <td>1.965730e-06</td>\n",
              "      <td>3.572866e-05</td>\n",
              "      <td>3.728085e-06</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9711</th>\n",
              "      <td>1.174115e-10</td>\n",
              "      <td>1.138450e-12</td>\n",
              "      <td>1.333672e-11</td>\n",
              "      <td>2.531653e-11</td>\n",
              "      <td>3.320499e-10</td>\n",
              "      <td>6.379326e-08</td>\n",
              "      <td>1.224576e-13</td>\n",
              "      <td>3.308441e-11</td>\n",
              "      <td>2.797492e-12</td>\n",
              "      <td>3.663664e-10</td>\n",
              "      <td>6.682804e-14</td>\n",
              "      <td>2.388729e-10</td>\n",
              "      <td>7.192675e-16</td>\n",
              "      <td>2.981313e-12</td>\n",
              "      <td>1.081923e-12</td>\n",
              "      <td>1.974088e-11</td>\n",
              "      <td>3.086215e-10</td>\n",
              "      <td>2.851866e-10</td>\n",
              "      <td>2.318276e-14</td>\n",
              "      <td>5.558960e-12</td>\n",
              "      <td>9.640536e-08</td>\n",
              "      <td>2.830290e-12</td>\n",
              "      <td>3.234425e-13</td>\n",
              "      <td>2.648493e-14</td>\n",
              "      <td>1.109922e-07</td>\n",
              "      <td>5.020579e-12</td>\n",
              "      <td>5.500836e-11</td>\n",
              "      <td>1.328590e-10</td>\n",
              "      <td>7.482698e-11</td>\n",
              "      <td>9.821940e-09</td>\n",
              "      <td>1.994418e-08</td>\n",
              "      <td>2.002466e-10</td>\n",
              "      <td>1.650054e-11</td>\n",
              "      <td>6.052429e-12</td>\n",
              "      <td>3.587613e-13</td>\n",
              "      <td>5.167047e-10</td>\n",
              "      <td>1.333829e-13</td>\n",
              "      <td>4.994437e-12</td>\n",
              "      <td>4.177162e-10</td>\n",
              "      <td>1.630393e-09</td>\n",
              "      <td>...</td>\n",
              "      <td>6.449170e-12</td>\n",
              "      <td>3.384656e-08</td>\n",
              "      <td>5.435725e-10</td>\n",
              "      <td>1.825190e-11</td>\n",
              "      <td>2.143211e-15</td>\n",
              "      <td>1.398237e-11</td>\n",
              "      <td>2.235430e-10</td>\n",
              "      <td>3.346979e-12</td>\n",
              "      <td>1.705171e-12</td>\n",
              "      <td>8.110619e-13</td>\n",
              "      <td>1.641357e-13</td>\n",
              "      <td>1.284663e-12</td>\n",
              "      <td>6.440242e-13</td>\n",
              "      <td>8.397542e-10</td>\n",
              "      <td>8.139177e-10</td>\n",
              "      <td>5.285636e-14</td>\n",
              "      <td>3.555187e-10</td>\n",
              "      <td>3.249230e-12</td>\n",
              "      <td>1.998440e-11</td>\n",
              "      <td>3.936571e-09</td>\n",
              "      <td>4.796320e-12</td>\n",
              "      <td>9.067831e-13</td>\n",
              "      <td>2.133861e-13</td>\n",
              "      <td>9.712746e-14</td>\n",
              "      <td>2.370614e-11</td>\n",
              "      <td>5.489531e-13</td>\n",
              "      <td>1.466913e-12</td>\n",
              "      <td>8.866620e-12</td>\n",
              "      <td>1.667712e-12</td>\n",
              "      <td>2.768725e-13</td>\n",
              "      <td>1.972322e-11</td>\n",
              "      <td>1.898695e-09</td>\n",
              "      <td>4.173691e-09</td>\n",
              "      <td>1.473939e-12</td>\n",
              "      <td>8.230886e-12</td>\n",
              "      <td>2.901602e-11</td>\n",
              "      <td>6.231759e-13</td>\n",
              "      <td>3.839918e-10</td>\n",
              "      <td>4.546474e-11</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>9712 rows × 1025 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                 0             1  ...          1023  label\n",
              "0     3.575464e-07  3.161658e-07  ...  3.311136e-06      5\n",
              "1     1.599470e-07  3.906797e-11  ...  1.360501e-08      3\n",
              "2     9.814446e-12  5.079448e-11  ...  1.850572e-10      2\n",
              "3     2.105866e-08  4.769444e-08  ...  1.000970e-08      5\n",
              "4     1.778951e-04  1.766992e-04  ...  7.318950e-04      3\n",
              "...            ...           ...  ...           ...    ...\n",
              "9707  3.766673e-04  1.946348e-07  ...  9.895754e-07      2\n",
              "9708  2.311359e-05  2.485752e-07  ...  9.300837e-06      2\n",
              "9709  2.240841e-07  9.109556e-10  ...  1.320440e-11      3\n",
              "9710  6.114797e-06  7.439993e-06  ...  3.728085e-06      5\n",
              "9711  1.174115e-10  1.138450e-12  ...  4.546474e-11      6\n",
              "\n",
              "[9712 rows x 1025 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MklcpqkURTFZ",
        "colab_type": "code",
        "outputId": "4d03bdec-8a19-418f-b516-acf340caffe9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "#fits SVM on vectorized training dataset\n",
        "clf2 = SVC(kernel='linear')\n",
        "clf2.fit(train_df.drop(('label'),axis=1),np.array(train_df.label,dtype=int))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
              "    decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
              "    kernel='linear', max_iter=-1, probability=False, random_state=None,\n",
              "    shrinking=True, tol=0.001, verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 107
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGb9c8YlRicP",
        "colab_type": "code",
        "outputId": "60b39ff6-48bc-4ce9-902c-0b37f3b6e825",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y_pred = clf2.predict(test_df.drop(('label'),axis=1))\n",
        "printScore(y_pred,np.array(test_df.label,dtype=int))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 21.8%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PzNFMVSGSSGd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}